From 65e97bc31ede2e40ce029a81dc3a885dc367f7f4 Mon Sep 17 00:00:00 2001 From: Narcis Beleuzu Date: Fri, 17 May 2019 11:30:27 +0300 Subject: [PATCH] Backed out 6 changesets (bug 1542963) for causing Bug 1552400. a=backout Backed out changeset 70fbe1a158ae (bug 1542963) Backed out changeset a1a84e0feabb (bug 1542963) Backed out changeset 14aa1bd254a4 (bug 1542963) Backed out changeset 11a714f491d5 (bug 1542963) Backed out changeset 2715bac40d2f (bug 1542963) Backed out changeset 8f69c7eeb6fd (bug 1542963) --HG-- extra : histedit_source : 5903adc5313d6af4fdafb40ae55aaa79856f3726 --- .flake8 | 4 +- python/mozbuild/mozbuild/action/buildlist.py | 41 +- .../mozbuild/mozbuild/action/check_binary.py | 9 +- .../mozbuild/action/download_wpt_manifest.py | 3 +- python/mozbuild/mozbuild/action/dump_env.py | 2 - .../mozbuild/mozbuild/action/dumpsymbols.py | 7 +- .../mozbuild/action/exe_7z_archive.py | 10 +- .../mozbuild/action/exe_7z_extract.py | 5 +- .../mozbuild/mozbuild/action/file_generate.py | 3 +- .../mozbuild/action/generate_searchjson.py | 65 +- .../mozbuild/action/generate_strings_xml.py | 2 +- .../action/generate_suggestedsites.py | 22 +- python/mozbuild/mozbuild/action/jar_maker.py | 2 +- .../mozbuild/action/langpack_manifest.py | 5 +- python/mozbuild/mozbuild/action/make_dmg.py | 2 +- python/mozbuild/mozbuild/action/make_unzip.py | 5 +- python/mozbuild/mozbuild/action/make_zip.py | 5 +- .../action/output_searchplugins_list.py | 20 +- .../mozbuild/action/package_fennec_apk.py | 3 +- .../action/package_generated_sources.py | 6 +- .../mozbuild/mozbuild/action/preprocessor.py | 5 +- .../mozbuild/action/process_define_files.py | 10 +- .../action/process_install_manifest.py | 33 +- .../mozbuild/action/symbols_archive.py | 6 +- .../mozbuild/mozbuild/action/test_archive.py | 2 +- python/mozbuild/mozbuild/action/tooltool.py | 7 +- python/mozbuild/mozbuild/action/unpack_dmg.py | 2 +- python/mozbuild/mozbuild/action/webidl.py | 2 +- python/mozbuild/mozbuild/action/wrap_rustc.py | 3 - python/mozbuild/mozbuild/action/xpccheck.py | 113 ++- .../mozbuild/mozbuild/action/xpidl-process.py | 27 +- python/mozbuild/mozbuild/action/zip.py | 2 +- python/mozbuild/mozbuild/analyze/graph.py | 15 +- python/mozbuild/mozbuild/analyze/hg.py | 29 +- .../mozbuild/mozbuild/android_version_code.py | 11 +- python/mozbuild/mozbuild/artifact_builds.py | 2 - python/mozbuild/mozbuild/artifacts.py | 140 ++-- python/mozbuild/mozbuild/backend/__init__.py | 2 - python/mozbuild/mozbuild/backend/base.py | 2 +- .../mozbuild/backend/cargo_build_defs.py | 2 +- python/mozbuild/mozbuild/backend/common.py | 25 +- .../mozbuild/backend/configenvironment.py | 45 +- .../mozbuild/mozbuild/backend/cpp_eclipse.py | 133 ++-- .../mozbuild/mozbuild/backend/fastermake.py | 19 +- .../mozbuild/backend/mach_commands.py | 10 +- .../mozbuild/backend/recursivemake.py | 144 ++-- .../mozbuild/backend/test_manifest.py | 2 +- python/mozbuild/mozbuild/backend/tup.py | 96 +-- .../mozbuild/mozbuild/backend/visualstudio.py | 56 +- python/mozbuild/mozbuild/base.py | 84 +-- python/mozbuild/mozbuild/chunkify.py | 3 +- .../mozbuild/codecoverage/chrome_map.py | 16 +- .../mozbuild/codecoverage/lcov_rewriter.py | 102 +-- .../mozbuild/codecoverage/manifest_handler.py | 2 - .../mozbuild/codecoverage/packager.py | 3 - .../mozbuild/compilation/codecomplete.py | 10 +- .../mozbuild/mozbuild/compilation/database.py | 9 +- python/mozbuild/mozbuild/compilation/util.py | 12 +- .../mozbuild/mozbuild/compilation/warnings.py | 16 +- python/mozbuild/mozbuild/config_status.py | 11 +- .../mozbuild/mozbuild/configure/__init__.py | 6 +- .../mozbuild/configure/check_debug_ranges.py | 7 +- .../mozbuild/mozbuild/configure/libstdcxx.py | 13 +- python/mozbuild/mozbuild/configure/lint.py | 2 +- .../mozbuild/mozbuild/configure/lint_util.py | 2 +- python/mozbuild/mozbuild/configure/options.py | 2 - python/mozbuild/mozbuild/configure/util.py | 9 +- .../mozbuild/mozbuild/controller/building.py | 72 +- .../mozbuild/mozbuild/controller/clobber.py | 13 +- python/mozbuild/mozbuild/doctor.py | 7 +- python/mozbuild/mozbuild/dotproperties.py | 5 +- .../mozbuild/export_telemetry_schema.py | 2 +- python/mozbuild/mozbuild/faster_daemon.py | 6 +- python/mozbuild/mozbuild/frontend/context.py | 425 +++++------ python/mozbuild/mozbuild/frontend/data.py | 46 +- python/mozbuild/mozbuild/frontend/emitter.py | 215 +++--- .../mozbuild/mozbuild/frontend/gyp_reader.py | 92 ++- .../mozbuild/frontend/mach_commands.py | 9 +- python/mozbuild/mozbuild/frontend/reader.py | 48 +- python/mozbuild/mozbuild/frontend/sandbox.py | 10 +- python/mozbuild/mozbuild/gen_test_backend.py | 2 - python/mozbuild/mozbuild/generated_sources.py | 2 - python/mozbuild/mozbuild/gn_processor.py | 15 +- python/mozbuild/mozbuild/html_build_viewer.py | 2 +- python/mozbuild/mozbuild/jar.py | 49 +- python/mozbuild/mozbuild/mach_commands.py | 710 ++++++++---------- python/mozbuild/mozbuild/makeutil.py | 7 +- python/mozbuild/mozbuild/moz_yaml.py | 1 - python/mozbuild/mozbuild/mozconfig.py | 19 +- python/mozbuild/mozbuild/mozinfo.py | 6 +- python/mozbuild/mozbuild/nodeutil.py | 2 +- python/mozbuild/mozbuild/preprocessor.py | 76 +- python/mozbuild/mozbuild/pythonutil.py | 2 +- .../mozbuild/repackaging/application_ini.py | 2 - python/mozbuild/mozbuild/repackaging/dmg.py | 5 +- .../mozbuild/repackaging/installer.py | 5 +- python/mozbuild/mozbuild/repackaging/mar.py | 4 +- python/mozbuild/mozbuild/repackaging/msi.py | 19 +- python/mozbuild/mozbuild/shellutil.py | 4 - python/mozbuild/mozbuild/sphinx.py | 10 +- python/mozbuild/mozbuild/telemetry.py | 1 - .../mozbuild/test/action/test_buildlist.py | 137 ++-- .../test/action/test_langpack_manifest.py | 2 - .../mozbuild/test/action/test_node.py | 3 +- .../test/action/test_package_fennec_apk.py | 2 +- .../action/test_process_install_manifest.py | 10 +- .../mozbuild/test/analyze/test_graph.py | 74 +- .../mozbuild/mozbuild/test/backend/common.py | 2 +- .../mozbuild/test/backend/test_build.py | 7 +- .../test/backend/test_configenvironment.py | 24 +- .../mozbuild/test/backend/test_fastermake.py | 11 +- .../test/backend/test_gn_processor.py | 4 +- .../backend/test_partialconfigenvironment.py | 15 +- .../test/backend/test_recursivemake.py | 82 +- .../test/backend/test_test_manifest.py | 2 +- .../test/backend/test_visualstudio.py | 4 +- .../test/codecoverage/test_lcov_rewrite.py | 38 +- python/mozbuild/mozbuild/test/common.py | 4 +- .../test/compilation/test_warnings.py | 8 +- .../mozbuild/test/configure/common.py | 3 +- .../mozbuild/mozbuild/test/configure/lint.py | 1 + .../test/configure/test_checks_configure.py | 16 +- .../test/configure/test_compile_checks.py | 3 +- .../mozbuild/test/configure/test_configure.py | 66 +- .../mozbuild/test/configure/test_lint.py | 1 + .../test/configure/test_moz_configure.py | 6 +- .../mozbuild/test/configure/test_options.py | 14 +- .../test/configure/test_toolchain_helpers.py | 3 - .../configure/test_toolkit_moz_configure.py | 11 +- .../mozbuild/test/configure/test_util.py | 15 +- .../test/controller/test_ccachestats.py | 3 +- .../mozbuild/test/controller/test_clobber.py | 5 +- .../mozbuild/test/frontend/test_context.py | 28 +- .../mozbuild/test/frontend/test_emitter.py | 119 ++- .../mozbuild/test/frontend/test_namespaces.py | 6 +- .../mozbuild/test/frontend/test_reader.py | 49 +- .../mozbuild/test/frontend/test_sandbox.py | 31 +- .../test/test_android_version_code.py | 44 +- .../mozbuild/test/test_artifact_cache.py | 3 +- python/mozbuild/mozbuild/test/test_base.py | 13 +- .../mozbuild/mozbuild/test/test_containers.py | 19 +- .../mozbuild/test/test_dotproperties.py | 12 +- .../mozbuild/mozbuild/test/test_expression.py | 115 ++- .../mozbuild/mozbuild/test/test_jarmaker.py | 52 +- .../mozbuild/mozbuild/test/test_licenses.py | 48 +- .../mozbuild/test/test_line_endings.py | 59 +- .../mozbuild/mozbuild/test/test_makeutil.py | 45 +- .../mozbuild/mozbuild/test/test_manifest.py | 2 +- .../mozbuild/mozbuild/test/test_mozconfig.py | 15 +- python/mozbuild/mozbuild/test/test_mozinfo.py | 3 - .../mozbuild/test/test_preprocessor.py | 2 - .../mozbuild/mozbuild/test/test_pythonutil.py | 2 - python/mozbuild/mozbuild/test/test_util.py | 38 +- python/mozbuild/mozbuild/testing.py | 20 +- python/mozbuild/mozbuild/util.py | 54 +- python/mozbuild/mozbuild/vendor_aom.py | 7 +- python/mozbuild/mozbuild/vendor_dav1d.py | 12 +- python/mozbuild/mozbuild/vendor_python.py | 3 +- python/mozbuild/mozbuild/vendor_rust.py | 54 +- python/mozbuild/mozbuild/virtualenv.py | 51 +- python/mozbuild/mozpack/archive.py | 2 +- python/mozbuild/mozpack/chrome/flags.py | 2 +- python/mozbuild/mozpack/chrome/manifest.py | 2 +- python/mozbuild/mozpack/copier.py | 2 +- python/mozbuild/mozpack/dmg.py | 2 - python/mozbuild/mozpack/errors.py | 4 +- python/mozbuild/mozpack/executables.py | 2 +- python/mozbuild/mozpack/files.py | 2 +- python/mozbuild/mozpack/hg.py | 2 +- python/mozbuild/mozpack/manifests.py | 2 +- python/mozbuild/mozpack/mozjar.py | 2 +- python/mozbuild/mozpack/packager/__init__.py | 2 +- python/mozbuild/mozpack/packager/formats.py | 2 +- python/mozbuild/mozpack/packager/l10n.py | 2 +- python/mozbuild/mozpack/packager/unpack.py | 2 +- python/mozbuild/mozpack/path.py | 2 +- .../mozpack/test/support/minify_js_verify.py | 2 +- python/mozbuild/mozpack/test/test_archive.py | 2 +- .../mozpack/test/test_chrome_flags.py | 2 - .../mozpack/test/test_chrome_manifest.py | 2 - python/mozbuild/mozpack/test/test_copier.py | 2 - python/mozbuild/mozpack/test/test_errors.py | 2 - python/mozbuild/mozpack/test/test_files.py | 2 - .../mozbuild/mozpack/test/test_manifests.py | 2 +- python/mozbuild/mozpack/test/test_mozjar.py | 2 - python/mozbuild/mozpack/test/test_packager.py | 2 - .../mozpack/test/test_packager_formats.py | 2 - .../mozpack/test/test_packager_l10n.py | 2 - .../mozpack/test/test_packager_unpack.py | 2 - python/mozbuild/mozpack/test/test_path.py | 2 - 190 files changed, 2092 insertions(+), 2820 deletions(-) diff --git a/.flake8 b/.flake8 index a715913c111e..05be05a88af8 100644 --- a/.flake8 +++ b/.flake8 @@ -25,6 +25,7 @@ exclude = python/devtools/migrate-l10n/migrate/main.py, python/l10n/fluent_migrations, python/mozbuild/dumbmake, + python/mozbuild/mozbuild, servo/components/style, testing/jsshell/benchmark.py, testing/marionette/mach_commands.py, @@ -68,7 +69,6 @@ exclude = memory/moz.configure, mobile/android/*.configure, node_modules, - python/mozbuild/mozbuild/test/configure/data, security/nss/, testing/marionette/harness/marionette_harness/runner/mixins, testing/marionette/harness/marionette_harness/tests, @@ -92,8 +92,6 @@ ignore = per-file-ignores = ipc/ipdl/*: F403, F405 - # cpp_eclipse has a lot of multi-line embedded XML which exceeds line length - python/mozbuild/mozbuild/backend/cpp_eclipse.py: E501 testing/firefox-ui/**/__init__.py: F401 testing/marionette/**/__init__.py: F401 testing/mozharness/configs/*: E124, E127, E128, E131, E231, E261, E265, E266, E501, W391 diff --git a/python/mozbuild/mozbuild/action/buildlist.py b/python/mozbuild/mozbuild/action/buildlist.py index ab9b90bbf125..9d601d69a90f 100644 --- a/python/mozbuild/mozbuild/action/buildlist.py +++ b/python/mozbuild/mozbuild/action/buildlist.py @@ -17,33 +17,32 @@ from mozbuild.util import ( lock_file, ) - def addEntriesToListFile(listFile, entries): - """Given a file |listFile| containing one entry per line, - add each entry in |entries| to the file, unless it is already - present.""" - ensureParentDir(listFile) - lock = lock_file(listFile + ".lck") - try: - if os.path.exists(listFile): - f = open(listFile) - existing = set(x.strip() for x in f.readlines()) - f.close() - else: - existing = set() - for e in entries: - if e not in existing: - existing.add(e) - with open(listFile, 'wb') as f: - f.write("\n".join(sorted(existing))+"\n") - finally: - del lock # Explicitly release the lock_file to free it + """Given a file |listFile| containing one entry per line, + add each entry in |entries| to the file, unless it is already + present.""" + ensureParentDir(listFile) + lock = lock_file(listFile + ".lck") + try: + if os.path.exists(listFile): + f = open(listFile) + existing = set(x.strip() for x in f.readlines()) + f.close() + else: + existing = set() + for e in entries: + if e not in existing: + existing.add(e) + with open(listFile, 'wb') as f: + f.write("\n".join(sorted(existing))+"\n") + finally: + lock = None def main(args): if len(args) < 2: print("Usage: buildlist.py [ ...]", - file=sys.stderr) + file=sys.stderr) return 1 return addEntriesToListFile(args[0], args[1:]) diff --git a/python/mozbuild/mozbuild/action/check_binary.py b/python/mozbuild/mozbuild/action/check_binary.py index 90f16da0c1dd..b00e36d742ce 100644 --- a/python/mozbuild/mozbuild/action/check_binary.py +++ b/python/mozbuild/mozbuild/action/check_binary.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import print_function, unicode_literals import argparse import os @@ -280,14 +280,13 @@ def check_networking(binary): s = 'TEST-UNEXPECTED-FAIL | check_networking | {} | Identified {} ' + \ 'networking function(s) being imported in the rust static library ({})' print(s.format(basename, len(bad_occurences_names), - ",".join(sorted(bad_occurences_names))), - file=sys.stderr) + ",".join(sorted(bad_occurences_names))), + file=sys.stderr) retcode = 1 elif buildconfig.substs.get('MOZ_AUTOMATION'): print('TEST-PASS | check_networking | {}'.format(basename)) return retcode - def checks(target, binary): # The clang-plugin is built as target but is really a host binary. # Cheat and pretend we were passed the right argument. @@ -346,7 +345,7 @@ def main(args): if options.networking and options.host: print('--networking is only valid with --target', - file=sys.stderr) + file=sys.stderr) return 1 if options.networking: diff --git a/python/mozbuild/mozbuild/action/download_wpt_manifest.py b/python/mozbuild/mozbuild/action/download_wpt_manifest.py index 7bb190948272..ffce7ba9e2b9 100644 --- a/python/mozbuild/mozbuild/action/download_wpt_manifest.py +++ b/python/mozbuild/mozbuild/action/download_wpt_manifest.py @@ -4,8 +4,7 @@ # This action is used to generate the wpt manifest -from __future__ import absolute_import, print_function - +import os import sys import buildconfig diff --git a/python/mozbuild/mozbuild/action/dump_env.py b/python/mozbuild/mozbuild/action/dump_env.py index 4e0642c65800..83e420d682fb 100644 --- a/python/mozbuild/mozbuild/action/dump_env.py +++ b/python/mozbuild/mozbuild/action/dump_env.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - # We invoke a Python program to dump our environment in order to get # native paths printed on Windows so that these paths can be incorporated # into Python configure's environment. diff --git a/python/mozbuild/mozbuild/action/dumpsymbols.py b/python/mozbuild/mozbuild/action/dumpsymbols.py index 0b6e8bd2f4de..ef5f179331e4 100644 --- a/python/mozbuild/mozbuild/action/dumpsymbols.py +++ b/python/mozbuild/mozbuild/action/dumpsymbols.py @@ -11,7 +11,6 @@ import shutil import sys import os - def dump_symbols(target, tracking_file, count_ctors=False): # Our tracking file, if present, will contain path(s) to the previously generated # symbols. Remove them in this case so we don't simply accumulate old symbols @@ -59,9 +58,8 @@ def dump_symbols(target, tracking_file, count_ctors=False): if objcopy: os.environ['OBJCOPY'] = objcopy - args = ([buildconfig.substs['PYTHON'], - os.path.join(buildconfig.topsrcdir, 'toolkit', - 'crashreporter', 'tools', 'symbolstore.py')] + + args = ([buildconfig.substs['PYTHON'], os.path.join(buildconfig.topsrcdir, 'toolkit', + 'crashreporter', 'tools', 'symbolstore.py')] + sym_store_args + ['-s', buildconfig.topsrcdir, dump_syms_bin, os.path.join(buildconfig.topobjdir, 'dist', @@ -75,7 +73,6 @@ def dump_symbols(target, tracking_file, count_ctors=False): fh.write(out_files) fh.flush() - def main(argv): parser = argparse.ArgumentParser( usage="Usage: dumpsymbols.py ") diff --git a/python/mozbuild/mozbuild/action/exe_7z_archive.py b/python/mozbuild/mozbuild/action/exe_7z_archive.py index 89defb1731b1..361a4ecfde35 100644 --- a/python/mozbuild/mozbuild/action/exe_7z_archive.py +++ b/python/mozbuild/mozbuild/action/exe_7z_archive.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function import os import shutil @@ -13,7 +13,6 @@ import mozpack.path as mozpath import buildconfig from mozbuild.base import BuildEnvironmentNotFoundException - def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): tmpdir = tempfile.mkdtemp(prefix='tmp') try: @@ -31,10 +30,7 @@ def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): except BuildEnvironmentNotFoundException: # configure hasn't been run, just use the default sevenz = '7z' - subprocess.check_call([ - sevenz, 'a', '-r', '-t7z', mozpath.join(tmpdir, 'app.7z'), '-mx', - '-m0=BCJ2', '-m1=LZMA:d25', '-m2=LZMA:d19', '-m3=LZMA:d19', '-mb0:1', - '-mb0s1:2', '-mb0s2:3']) + subprocess.check_call([sevenz, 'a', '-r', '-t7z', mozpath.join(tmpdir, 'app.7z'), '-mx', '-m0=BCJ2', '-m1=LZMA:d25', '-m2=LZMA:d19', '-m3=LZMA:d19', '-mb0:1', '-mb0s1:2', '-mb0s2:3']) with open(package, 'wb') as o: for i in [final_sfx, tagfile, mozpath.join(tmpdir, 'app.7z')]: @@ -45,7 +41,6 @@ def archive_exe(pkg_dir, tagfile, sfx_package, package, use_upx): shutil.move('core', pkg_dir) shutil.rmtree(tmpdir) - def main(args): if len(args) != 4: print('Usage: exe_7z_archive.py ', @@ -55,6 +50,5 @@ def main(args): archive_exe(args[0], args[1], args[2], args[3], args[4]) return 0 - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/exe_7z_extract.py b/python/mozbuild/mozbuild/action/exe_7z_extract.py index eecc211667db..db9a8fd59360 100644 --- a/python/mozbuild/mozbuild/action/exe_7z_extract.py +++ b/python/mozbuild/mozbuild/action/exe_7z_extract.py @@ -2,18 +2,16 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function import shutil import sys import subprocess - def extract_exe(package, target): subprocess.check_call(['7z', 'x', package, 'core']) shutil.move('core', target) - def main(args): if len(args) != 2: print('Usage: exe_7z_extract.py ', @@ -23,6 +21,5 @@ def main(args): extract_exe(args[0], args[1]) return 0 - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/file_generate.py b/python/mozbuild/mozbuild/action/file_generate.py index edad180c9dae..bb52c1ca5d17 100644 --- a/python/mozbuild/mozbuild/action/file_generate.py +++ b/python/mozbuild/mozbuild/action/file_generate.py @@ -69,7 +69,7 @@ def main(argv): with FileAvoidWrite(args.output_file, mode='rb') as output: try: ret = module.__dict__[method](output, *args.additional_arguments, **kwargs) - except Exception: + except: # Ensure that we don't overwrite the file if the script failed. output.avoid_writing_to_file() raise @@ -116,6 +116,5 @@ def main(argv): return 1 return ret - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/generate_searchjson.py b/python/mozbuild/mozbuild/action/generate_searchjson.py index aeb4b986cead..8dd85552cf7e 100644 --- a/python/mozbuild/mozbuild/action/generate_searchjson.py +++ b/python/mozbuild/mozbuild/action/generate_searchjson.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import sys import json import copy @@ -16,61 +14,54 @@ output_file = sys.argv[3] output = open(output_file, 'w') with open(sys.argv[1]) as f: - searchinfo = json.load(f) + searchinfo = json.load(f) # If we have a locale, use it, otherwise use the default if locale in searchinfo["locales"]: - localeSearchInfo = searchinfo["locales"][locale] + localeSearchInfo = searchinfo["locales"][locale] else: - localeSearchInfo = {} - localeSearchInfo["default"] = searchinfo["default"] - + localeSearchInfo = {} + localeSearchInfo["default"] = searchinfo["default"] def validateDefault(key): - if key not in searchinfo["default"]: - print("Error: Missing default %s in list.json" % (key), file=sys.stderr) - sys.exit(1) + if (not key in searchinfo["default"]): + print >>sys.stderr, "Error: Missing default %s in list.json" % (key) + sys.exit(1) - -validateDefault("searchDefault") -validateDefault("visibleDefaultEngines") +validateDefault("searchDefault"); +validateDefault("visibleDefaultEngines"); # If the selected locale doesn't have a searchDefault, # use the global one. -if "searchDefault" not in localeSearchInfo["default"]: - localeSearchInfo["default"]["searchDefault"] = searchinfo["default"]["searchDefault"] +if not "searchDefault" in localeSearchInfo["default"]: + localeSearchInfo["default"]["searchDefault"] = searchinfo["default"]["searchDefault"] # If the selected locale doesn't have a searchOrder, # use the global one if present. # searchOrder is NOT required. -if ( - "searchOrder" not in localeSearchInfo["default"] - and "searchOrder" in searchinfo["default"] -): +if not "searchOrder" in localeSearchInfo["default"] and "searchOrder" in searchinfo["default"]: localeSearchInfo["default"]["searchOrder"] = searchinfo["default"]["searchOrder"] # If we have region overrides, enumerate through them # and add the additional regions to the locale information. if "regionOverrides" in searchinfo: - regionOverrides = searchinfo["regionOverrides"] + regionOverrides = searchinfo["regionOverrides"] - for region in regionOverrides: - # Only add a new engine list if there is an engine that is overridden - enginesToOverride = set(regionOverrides[region].keys()) - if region in localeSearchInfo and "visibleDefaultEngines" in localeSearchInfo[region]: - visibleDefaultEngines = localeSearchInfo[region]["visibleDefaultEngines"] - else: - visibleDefaultEngines = localeSearchInfo["default"]["visibleDefaultEngines"] - if set(visibleDefaultEngines) & enginesToOverride: - if region not in localeSearchInfo: - localeSearchInfo[region] = {} - localeSearchInfo[region]["visibleDefaultEngines"] = copy.deepcopy( - visibleDefaultEngines) - for i, engine in enumerate(localeSearchInfo[region]["visibleDefaultEngines"]): - if engine in regionOverrides[region]: - localeSearchInfo[region]["visibleDefaultEngines"][i] = \ - regionOverrides[region][engine] + for region in regionOverrides: + # Only add a new engine list if there is an engine that is overridden + enginesToOverride = set(regionOverrides[region].keys()) + if region in localeSearchInfo and "visibleDefaultEngines" in localeSearchInfo[region]: + visibleDefaultEngines = localeSearchInfo[region]["visibleDefaultEngines"] + else: + visibleDefaultEngines = localeSearchInfo["default"]["visibleDefaultEngines"] + if set(visibleDefaultEngines) & enginesToOverride: + if region not in localeSearchInfo: + localeSearchInfo[region] = {} + localeSearchInfo[region]["visibleDefaultEngines"] = copy.deepcopy(visibleDefaultEngines) + for i, engine in enumerate(localeSearchInfo[region]["visibleDefaultEngines"]): + if engine in regionOverrides[region]: + localeSearchInfo[region]["visibleDefaultEngines"][i] = regionOverrides[region][engine] output.write(json.dumps(localeSearchInfo, ensure_ascii=False).encode('utf8')) -output.close() +output.close(); diff --git a/python/mozbuild/mozbuild/action/generate_strings_xml.py b/python/mozbuild/mozbuild/action/generate_strings_xml.py index 5f35b86b74ee..b60b480f0748 100644 --- a/python/mozbuild/mozbuild/action/generate_strings_xml.py +++ b/python/mozbuild/mozbuild/action/generate_strings_xml.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys diff --git a/python/mozbuild/mozbuild/action/generate_suggestedsites.py b/python/mozbuild/mozbuild/action/generate_suggestedsites.py index eae1c3f8b6dd..63300957805a 100644 --- a/python/mozbuild/mozbuild/action/generate_suggestedsites.py +++ b/python/mozbuild/mozbuild/action/generate_suggestedsites.py @@ -38,9 +38,13 @@ import os from mozbuild.dotproperties import ( DotProperties, ) +from mozbuild.util import ( + FileAvoidWrite, +) from mozpack.files import ( FileFinder, ) +import mozpack.path as mozpath def merge_properties(paths): @@ -88,8 +92,7 @@ def main(output, *args, **kwargs): properties = merge_properties(sources) # Keep these two in sync. - image_url_template = \ - 'android.resource://%s/drawable/suggestedsites_{name}' % opts.android_package_name + image_url_template = 'android.resource://%s/drawable/suggestedsites_{name}' % opts.android_package_name drawables_template = 'drawable*/suggestedsites_{name}.*' # Load properties corresponding to each site name and define their @@ -99,8 +102,7 @@ def main(output, *args, **kwargs): def add_names(names, defaults={}): for name in names: site = copy.deepcopy(defaults) - site.update(properties.get_dict('browser.suggestedsites.{name}'.format( - name=name), required_keys=('title', 'url', 'bgcolor'))) + site.update(properties.get_dict('browser.suggestedsites.{name}'.format(name=name), required_keys=('title', 'url', 'bgcolor'))) site['imageurl'] = image_url_template.format(name=name) sites.append(site) @@ -114,13 +116,11 @@ def main(output, *args, **kwargs): matches = [p for p, _ in finder.find(drawables_template.format(name=name))] if not matches: raise Exception("Could not find drawable in '{resources}' for '{name}'" - .format(resources=resources, name=name)) + .format(resources=resources, name=name)) else: if opts.verbose: print("Found {len} drawables in '{resources}' for '{name}': {matches}" - .format(len=len(matches), resources=resources, - name=name, matches=matches) - ) + .format(len=len(matches), resources=resources, name=name, matches=matches)) # We want the lists to be ordered for reproducibility. Each list has a # "default" JSON list item which will be extended by the properties read. @@ -129,14 +129,12 @@ def main(output, *args, **kwargs): ('browser.suggestedsites.restricted.list', {'restricted': True}), ] if opts.verbose: - print('Reading {len} suggested site lists: {lists}'.format( - len=len(lists), lists=[list_name for list_name, _ in lists])) + print('Reading {len} suggested site lists: {lists}'.format(len=len(lists), lists=[list_name for list_name, _ in lists])) for (list_name, list_item_defaults) in lists: names = properties.get_list(list_name) if opts.verbose: - print('Reading {len} suggested sites from {list}: {names}'.format( - len=len(names), list=list_name, names=names)) + print('Reading {len} suggested sites from {list}: {names}'.format(len=len(names), list=list_name, names=names)) add_names(names, list_item_defaults) # We must define at least one site -- that's what the fallback is for. diff --git a/python/mozbuild/mozbuild/action/jar_maker.py b/python/mozbuild/mozbuild/action/jar_maker.py index 48771948f433..3e3c3c83eff1 100644 --- a/python/mozbuild/mozbuild/action/jar_maker.py +++ b/python/mozbuild/mozbuild/action/jar_maker.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys diff --git a/python/mozbuild/mozbuild/action/langpack_manifest.py b/python/mozbuild/mozbuild/action/langpack_manifest.py index 6c3d9433b588..6f7269707a04 100644 --- a/python/mozbuild/mozbuild/action/langpack_manifest.py +++ b/python/mozbuild/mozbuild/action/langpack_manifest.py @@ -8,7 +8,7 @@ # the locale directory, chrome registry entries and other information # necessary to produce the complete manifest file for a language pack. ### -from __future__ import absolute_import, print_function +from __future__ import absolute_import import argparse import sys @@ -26,6 +26,7 @@ from mozpack.chrome.manifest import ( ) from mozbuild.configure.util import Version from mozbuild.preprocessor import Preprocessor +import buildconfig def write_file(path, content): @@ -307,7 +308,7 @@ def get_version_maybe_buildid(min_version): version = str(min_version) buildid = os.environ.get('MOZ_BUILD_DATE') if buildid and len(buildid) != 14: - print('Ignoring invalid MOZ_BUILD_DATE: %s' % buildid, file=sys.stderr) + print >>sys.stderr, 'Ignoring invalid MOZ_BUILD_DATE: %s' % buildid buildid = None if buildid: version = version + "buildid" + buildid diff --git a/python/mozbuild/mozbuild/action/make_dmg.py b/python/mozbuild/mozbuild/action/make_dmg.py index e5f3b7cfff93..80a47e7161f2 100644 --- a/python/mozbuild/mozbuild/action/make_dmg.py +++ b/python/mozbuild/mozbuild/action/make_dmg.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function from mozpack import dmg diff --git a/python/mozbuild/mozbuild/action/make_unzip.py b/python/mozbuild/mozbuild/action/make_unzip.py index f09f05cc9c13..5d8ee36ca6d6 100644 --- a/python/mozbuild/mozbuild/action/make_unzip.py +++ b/python/mozbuild/mozbuild/action/make_unzip.py @@ -2,16 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function import sys import subprocess - def make_unzip(package): subprocess.check_call(['unzip', package]) - def main(args): if len(args) != 1: print('Usage: make_unzip.py ', @@ -21,6 +19,5 @@ def main(args): make_unzip(args[0]) return 0 - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/make_zip.py b/python/mozbuild/mozbuild/action/make_zip.py index 3aafd27f019b..13176afd5e9f 100644 --- a/python/mozbuild/mozbuild/action/make_zip.py +++ b/python/mozbuild/mozbuild/action/make_zip.py @@ -2,16 +2,14 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function import sys import subprocess - def make_zip(source, package): subprocess.check_call(['zip', '-r9D', package, source, '-x', '\*/.mkdir.done']) - def main(args): if len(args) != 2: print('Usage: make_zip.py ', @@ -21,6 +19,5 @@ def main(args): make_zip(args[0], args[1]) return 0 - if __name__ == '__main__': sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/output_searchplugins_list.py b/python/mozbuild/mozbuild/action/output_searchplugins_list.py index 8a2aa3e609db..a4767bbe40be 100644 --- a/python/mozbuild/mozbuild/action/output_searchplugins_list.py +++ b/python/mozbuild/mozbuild/action/output_searchplugins_list.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import sys import json @@ -12,24 +10,24 @@ engines = [] locale = sys.argv[2] with open(sys.argv[1]) as f: - searchinfo = json.load(f) + searchinfo = json.load(f) # Get a list of the engines from the locale or the default engines = set() if locale in searchinfo["locales"]: - for region, table in searchinfo["locales"][locale].iteritems(): - if "visibleDefaultEngines" in table: - engines.update(table["visibleDefaultEngines"]) + for region, table in searchinfo["locales"][locale].iteritems(): + if "visibleDefaultEngines" in table: + engines.update(table["visibleDefaultEngines"]) if not engines: - engines.update(searchinfo["default"]["visibleDefaultEngines"]) + engines.update(searchinfo["default"]["visibleDefaultEngines"]) # Get additional engines from regionOverrides for region, overrides in searchinfo["regionOverrides"].iteritems(): - for originalengine, replacement in overrides.iteritems(): - if originalengine in engines: - # We add the engine because we still need the original - engines.add(replacement) + for originalengine, replacement in overrides.iteritems(): + if originalengine in engines: + # We add the engine because we still need the original + engines.add(replacement) # join() will take an iterable, not just a list. print('\n'.join(engines)) diff --git a/python/mozbuild/mozbuild/action/package_fennec_apk.py b/python/mozbuild/mozbuild/action/package_fennec_apk.py index e155b9779813..93b2a59be77f 100644 --- a/python/mozbuild/mozbuild/action/package_fennec_apk.py +++ b/python/mozbuild/mozbuild/action/package_fennec_apk.py @@ -11,6 +11,7 @@ from __future__ import absolute_import, print_function import argparse import buildconfig import os +import subprocess import sys from mozpack.copier import Jarrer @@ -69,7 +70,7 @@ def package_fennec_apk(inputs=[], omni_ja=None, if verbose: print('Packaging %s from %s' % (path, file.path)) if not os.path.exists(abspath): - raise ValueError('File %s not found (looked for %s)' % + raise ValueError('File %s not found (looked for %s)' % \ (file.path, abspath)) if jarrer.contains(path): jarrer.remove(path) diff --git a/python/mozbuild/mozbuild/action/package_generated_sources.py b/python/mozbuild/mozbuild/action/package_generated_sources.py index 466b6f77ef6e..1f4c35a65846 100644 --- a/python/mozbuild/mozbuild/action/package_generated_sources.py +++ b/python/mozbuild/mozbuild/action/package_generated_sources.py @@ -5,6 +5,8 @@ from __future__ import absolute_import, print_function, unicode_literals import argparse +import json +import os.path import sys import buildconfig @@ -21,15 +23,13 @@ def main(argv): args = parser.parse_args(argv) objdir_abspath = mozpath.abspath(buildconfig.topobjdir) - def is_valid_entry(entry): if isinstance(entry[1], BaseFile): entry_abspath = mozpath.abspath(entry[1].path) else: entry_abspath = mozpath.abspath(entry[1]) if not entry_abspath.startswith(objdir_abspath): - print("Warning: omitting generated source [%s] from archive" % entry_abspath, - file=sys.stderr) + print("Warning: omitting generated source [%s] from archive" % entry_abspath, file=sys.stderr) return False return True diff --git a/python/mozbuild/mozbuild/action/preprocessor.py b/python/mozbuild/mozbuild/action/preprocessor.py index 751af73949a7..78f86ed6e82e 100644 --- a/python/mozbuild/mozbuild/action/preprocessor.py +++ b/python/mozbuild/mozbuild/action/preprocessor.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys @@ -15,11 +15,10 @@ def generate(output, *args): pp.handleCommandLine(list(args), True) return set(pp.includes) - def main(args): pp = Preprocessor() pp.handleCommandLine(args, True) if __name__ == "__main__": - main(sys.argv[1:]) + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/process_define_files.py b/python/mozbuild/mozbuild/action/process_define_files.py index 2b846dd50ec3..f66bc68ad1a0 100644 --- a/python/mozbuild/mozbuild/action/process_define_files.py +++ b/python/mozbuild/mozbuild/action/process_define_files.py @@ -10,6 +10,7 @@ import re import sys from buildconfig import topsrcdir, topobjdir from mozbuild.backend.configenvironment import PartialConfigEnvironment +from mozbuild.util import FileAvoidWrite import mozpack.path as mozpath @@ -50,12 +51,11 @@ def process_define_file(output, input): raise Exception( '`#define ALLDEFINES` is not allowed in a ' 'CONFIGURE_DEFINE_FILE') - + # WebRTC files like to define WINVER and _WIN32_WINNT + # via the command line, which raises a mass of macro + # redefinition warnings. Just handle those macros + # specially here. def define_for_name(name, val): - """WebRTC files like to define WINVER and _WIN32_WINNT - via the command line, which raises a mass of macro - redefinition warnings. Just handle those macros - specially here.""" define = "#define {name} {val}".format(name=name, val=val) if name in ('WINVER', '_WIN32_WINNT'): return '#if !defined({name})\n{define}\n#endif' \ diff --git a/python/mozbuild/mozbuild/action/process_install_manifest.py b/python/mozbuild/mozbuild/action/process_install_manifest.py index 16254fe8e681..3159d391b943 100644 --- a/python/mozbuild/mozbuild/action/process_install_manifest.py +++ b/python/mozbuild/mozbuild/action/process_install_manifest.py @@ -29,8 +29,8 @@ COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \ def process_manifest(destdir, paths, track, - no_symlinks=False, - defines={}): + no_symlinks=False, + defines={}): if os.path.exists(track): # We use the same format as install manifests for the tracking @@ -44,15 +44,15 @@ def process_manifest(destdir, paths, track, for p, f in finder.find(dest): remove_unaccounted.add(p, dummy_file) - remove_empty_directories = True - remove_all_directory_symlinks = True + remove_empty_directories=True + remove_all_directory_symlinks=True else: # If tracking is enabled and there is no file, we don't want to # be removing anything. remove_unaccounted = False - remove_empty_directories = False - remove_all_directory_symlinks = False + remove_empty_directories=False + remove_all_directory_symlinks=False manifest = InstallManifest() for path in paths: @@ -64,9 +64,9 @@ def process_manifest(destdir, paths, track, copier, defines_override=defines, link_policy=link_policy ) result = copier.copy(destdir, - remove_unaccounted=remove_unaccounted, - remove_all_directory_symlinks=remove_all_directory_symlinks, - remove_empty_directories=remove_empty_directories) + remove_unaccounted=remove_unaccounted, + remove_all_directory_symlinks=remove_all_directory_symlinks, + remove_empty_directories=remove_empty_directories) if track: # We should record files that we actually copied. @@ -83,21 +83,21 @@ def main(argv): parser.add_argument('destdir', help='Destination directory.') parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).') parser.add_argument('--no-symlinks', action='store_true', - help='Do not install symbolic links. Always copy files') + help='Do not install symbolic links. Always copy files') parser.add_argument('--track', metavar="PATH", required=True, - help='Use installed files tracking information from the given path.') + help='Use installed files tracking information from the given path.') parser.add_argument('-D', action=DefinesAction, - dest='defines', metavar="VAR[=VAL]", - help='Define a variable to override what is specified in the manifest') + dest='defines', metavar="VAR[=VAL]", + help='Define a variable to override what is specified in the manifest') args = parser.parse_args(argv) start = time.time() result = process_manifest(args.destdir, args.manifests, - track=args.track, - no_symlinks=args.no_symlinks, - defines=args.defines) + track=args.track, + no_symlinks=args.no_symlinks, + defines=args.defines) elapsed = time.time() - start @@ -109,6 +109,5 @@ def main(argv): rm_files=result.removed_files_count, rm_dirs=result.removed_directories_count)) - if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/symbols_archive.py b/python/mozbuild/mozbuild/action/symbols_archive.py index 6555292f50c5..772ebf183d06 100644 --- a/python/mozbuild/mozbuild/action/symbols_archive.py +++ b/python/mozbuild/mozbuild/action/symbols_archive.py @@ -12,7 +12,6 @@ from mozpack.files import FileFinder from mozpack.mozjar import JarWriter import mozpack.path as mozpath - def make_archive(archive_name, base, exclude, include): compress = ['**/*.sym'] finder = FileFinder(base, ignore=exclude) @@ -28,13 +27,11 @@ def make_archive(archive_name, base, exclude, include): writer.add(p.encode('utf-8'), f, mode=f.mode, compress=should_compress, skip_duplicates=True) - def main(argv): parser = argparse.ArgumentParser(description='Produce a symbols archive') parser.add_argument('archive', help='Which archive to generate') parser.add_argument('base', help='Base directory to package') - parser.add_argument('--full-archive', action='store_true', - help='Generate a full symbol archive') + parser.add_argument('--full-archive', action='store_true', help='Generate a full symbol archive') args = parser.parse_args(argv) @@ -50,6 +47,5 @@ def main(argv): make_archive(args.archive, args.base, excludes, includes) - if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py index 388713c50f64..7b1d4781a7d8 100644 --- a/python/mozbuild/mozbuild/action/test_archive.py +++ b/python/mozbuild/mozbuild/action/test_archive.py @@ -642,7 +642,7 @@ for k, v in ARCHIVE_FILES.items(): continue ignores = set(itertools.chain(*(e.get('ignore', []) - for e in ARCHIVE_FILES['common']))) + for e in ARCHIVE_FILES['common']))) if not any(p.startswith('%s/' % k) for p in ignores): raise Exception('"common" ignore list probably should contain %s' % k) diff --git a/python/mozbuild/mozbuild/action/tooltool.py b/python/mozbuild/mozbuild/action/tooltool.py index 5882e67368a4..646b31a19019 100755 --- a/python/mozbuild/mozbuild/action/tooltool.py +++ b/python/mozbuild/mozbuild/action/tooltool.py @@ -17,8 +17,6 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. -from __future__ import absolute_import, print_function - # A manifest file specifies files in that directory that are stored # elsewhere. This file should only list files in the same directory # in which the manifest file resides and it should be called @@ -750,7 +748,7 @@ def _cache_checksum_matches(base_file, checksum): log.info("Cache matches, avoiding extracting in '%s'" % base_file) return True return False - except IOError: + except IOError as e: return False @@ -1024,7 +1022,7 @@ def _authorize(req, auth_file): try: auth_file_content = json.loads(auth_file_content) is_taskcluster_auth = True - except Exception: + except: pass if is_taskcluster_auth: @@ -1302,6 +1300,5 @@ def main(argv, _skip_logging=False): return 0 if process_command(options, args) else 1 - if __name__ == "__main__": # pragma: no cover sys.exit(main(sys.argv)) diff --git a/python/mozbuild/mozbuild/action/unpack_dmg.py b/python/mozbuild/mozbuild/action/unpack_dmg.py index abdf50a45846..5f66d98cf940 100644 --- a/python/mozbuild/mozbuild/action/unpack_dmg.py +++ b/python/mozbuild/mozbuild/action/unpack_dmg.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function from mozpack import dmg diff --git a/python/mozbuild/mozbuild/action/webidl.py b/python/mozbuild/mozbuild/action/webidl.py index 99d90acb285d..d595c728ec05 100644 --- a/python/mozbuild/mozbuild/action/webidl.py +++ b/python/mozbuild/mozbuild/action/webidl.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys diff --git a/python/mozbuild/mozbuild/action/wrap_rustc.py b/python/mozbuild/mozbuild/action/wrap_rustc.py index 39893b174c25..e80a13661e1c 100644 --- a/python/mozbuild/mozbuild/action/wrap_rustc.py +++ b/python/mozbuild/mozbuild/action/wrap_rustc.py @@ -9,7 +9,6 @@ import subprocess import sys import os - def parse_outputs(crate_output, dep_outputs, pass_l_flag): env = {} args = [] @@ -60,7 +59,6 @@ def parse_outputs(crate_output, dep_outputs, pass_l_flag): return env, args - def wrap_rustc(args): parser = argparse.ArgumentParser() parser.add_argument('--crate-out', nargs='?') @@ -75,6 +73,5 @@ def wrap_rustc(args): os.environ.update(new_env) return subprocess.Popen(args.cmd + new_args, cwd=args.cwd).wait() - if __name__ == '__main__': sys.exit(wrap_rustc(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/xpccheck.py b/python/mozbuild/mozbuild/action/xpccheck.py index f8a1c918c0b3..c3170a8da8fd 100644 --- a/python/mozbuild/mozbuild/action/xpccheck.py +++ b/python/mozbuild/mozbuild/action/xpccheck.py @@ -8,93 +8,76 @@ corresponding .ini file. Usage: xpccheck.py [ ...] ''' -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys import os from glob import glob import manifestparser - def getIniTests(testdir): - mp = manifestparser.ManifestParser(strict=False) - mp.read(os.path.join(testdir, 'xpcshell.ini')) - return mp.tests - + mp = manifestparser.ManifestParser(strict=False) + mp.read(os.path.join(testdir, 'xpcshell.ini')) + return mp.tests def verifyDirectory(initests, directory): - files = glob(os.path.join(os.path.abspath(directory), "test_*")) - for f in files: - if (not os.path.isfile(f)): - continue + files = glob(os.path.join(os.path.abspath(directory), "test_*")) + for f in files: + if (not os.path.isfile(f)): + continue - name = os.path.basename(f) - if name.endswith('.in'): - name = name[:-3] + name = os.path.basename(f) + if name.endswith('.in'): + name = name[:-3] - if not name.endswith('.js'): - continue - - found = False - for test in initests: - if os.path.join(os.path.abspath(directory), name) == test['path']: - found = True - break - - if not found: - print(('TEST-UNEXPECTED-FAIL | xpccheck | test ' - '%s is missing from test manifest %s!') % ( - name, - os.path.join(directory, 'xpcshell.ini'), - ), - file=sys.stderr, - ) - sys.exit(1) + if not name.endswith('.js'): + continue + found = False + for test in initests: + if os.path.join(os.path.abspath(directory), name) == test['path']: + found = True + break + + if not found: + print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % (name, os.path.join(directory, 'xpcshell.ini')) + sys.exit(1) def verifyIniFile(initests, directory): - files = glob(os.path.join(os.path.abspath(directory), "test_*")) - for test in initests: - name = test['path'].split('/')[-1] + files = glob(os.path.join(os.path.abspath(directory), "test_*")) + for test in initests: + name = test['path'].split('/')[-1] - found = False - for f in files: + found = False + for f in files: - fname = f.split('/')[-1] - if fname.endswith('.in'): - fname = '.in'.join(fname.split('.in')[:-1]) + fname = f.split('/')[-1] + if fname.endswith('.in'): + fname = '.in'.join(fname.split('.in')[:-1]) - if os.path.join(os.path.abspath(directory), fname) == test['path']: - found = True - break - - if not found: - print(("TEST-UNEXPECTED-FAIL | xpccheck | found " - "%s in xpcshell.ini and not in directory '%s'") % ( - name, - directory, - ), - file=sys.stderr, - ) - sys.exit(1) + if os.path.join(os.path.abspath(directory), fname) == test['path']: + found = True + break + if not found: + print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | found %s in xpcshell.ini and not in directory '%s'" % (name, directory) + sys.exit(1) def main(argv): - if len(argv) < 2: - print("Usage: xpccheck.py [ ...]", - file=sys.stderr) - sys.exit(1) + if len(argv) < 2: + print >>sys.stderr, "Usage: xpccheck.py [ ...]" + sys.exit(1) - for d in argv[1:]: - # xpcshell-unpack is a copy of xpcshell sibling directory and in the Makefile - # we copy all files (including xpcshell.ini from the sibling directory. - if d.endswith('toolkit/mozapps/extensions/test/xpcshell-unpack'): - continue - - initests = getIniTests(d) - verifyDirectory(initests, d) - verifyIniFile(initests, d) + topsrcdir = argv[0] + for d in argv[1:]: + # xpcshell-unpack is a copy of xpcshell sibling directory and in the Makefile + # we copy all files (including xpcshell.ini from the sibling directory. + if d.endswith('toolkit/mozapps/extensions/test/xpcshell-unpack'): + continue + initests = getIniTests(d) + verifyDirectory(initests, d) + verifyIniFile(initests, d) if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/xpidl-process.py b/python/mozbuild/mozbuild/action/xpidl-process.py index d78dadb048a8..ae9ba873eed4 100755 --- a/python/mozbuild/mozbuild/action/xpidl-process.py +++ b/python/mozbuild/mozbuild/action/xpidl-process.py @@ -7,12 +7,14 @@ # input IDL file(s). It's purpose is to directly support the build # system. The API will change to meet the needs of the build system. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import argparse import os import sys +from io import BytesIO + from xpidl import jsonxpt from buildconfig import topsrcdir from xpidl.header import print_header @@ -85,33 +87,32 @@ def process(input_dirs, inc_paths, bindings_conf, cache_dir, header_dir, def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--cache-dir', - help='Directory in which to find or write cached lexer data.') + help='Directory in which to find or write cached lexer data.') parser.add_argument('--depsdir', - help='Directory in which to write dependency files.') + help='Directory in which to write dependency files.') parser.add_argument('--bindings-conf', - help='Path to the WebIDL binding configuration file.') + help='Path to the WebIDL binding configuration file.') parser.add_argument('--input-dir', dest='input_dirs', action='append', default=[], help='Directory(ies) in which to find source .idl files.') parser.add_argument('headerdir', - help='Directory in which to write header files.') + help='Directory in which to write header files.') parser.add_argument('xpcrsdir', - help='Directory in which to write rust xpcom binding files.') + help='Directory in which to write rust xpcom binding files.') parser.add_argument('xptdir', - help='Directory in which to write xpt file.') + help='Directory in which to write xpt file.') parser.add_argument('module', - help='Final module name to use for linked output xpt file.') + help='Final module name to use for linked output xpt file.') parser.add_argument('idls', nargs='+', - help='Source .idl file(s).') + help='Source .idl file(s).') parser.add_argument('-I', dest='incpath', action='append', default=[], - help='Extra directories where to look for included .idl files.') + help='Extra directories where to look for included .idl files.') args = parser.parse_args(argv) incpath = [os.path.join(topsrcdir, p) for p in args.incpath] process(args.input_dirs, incpath, args.bindings_conf, args.cache_dir, - args.headerdir, args.xpcrsdir, args.xptdir, args.depsdir, args.module, - args.idls) - + args.headerdir, args.xpcrsdir, args.xptdir, args.depsdir, args.module, + args.idls) if __name__ == '__main__': main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/zip.py b/python/mozbuild/mozbuild/action/zip.py index 86bb03ccaa23..837bed33213e 100644 --- a/python/mozbuild/mozbuild/action/zip.py +++ b/python/mozbuild/mozbuild/action/zip.py @@ -5,7 +5,7 @@ # This script creates a zip file, but will also strip any binaries # it finds before adding them to the zip. -from __future__ import absolute_import, print_function +from __future__ import absolute_import from mozpack.files import FileFinder from mozpack.copier import Jarrer diff --git a/python/mozbuild/mozbuild/analyze/graph.py b/python/mozbuild/mozbuild/analyze/graph.py index a0a25008f99e..5c5acee36888 100644 --- a/python/mozbuild/mozbuild/analyze/graph.py +++ b/python/mozbuild/mozbuild/analyze/graph.py @@ -2,12 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import sqlite3 as lite - class Node(object): def __init__(self, graph, node_id): @@ -42,12 +39,11 @@ class Node(object): return os.path.join(parent.get_path(graph), self.name) def calculate_mtime(self): - if self.type == 0: # only files have meaningful costs + if self.type == 0: # only files have meaningful costs return sum(x.mtime for x in self.cmds) else: return None - class Graph(object): def __init__(self, path=None, connect=None): @@ -71,7 +67,7 @@ class Graph(object): self.connect.close() def query_arg(self, q, arg): - assert isinstance(arg, tuple) # execute() requires tuple argument + assert isinstance(arg, tuple) #execute() requires tuple argument cursor = self.connect.cursor() cursor.execute(q, arg) return cursor @@ -94,7 +90,7 @@ class Graph(object): ret = self.query_arg('SELECT id FROM node \ WHERE dir=? AND name=?', (nodeid, part)).fetchone() # fetchone should be ok bc dir and and name combo is unique - if ret is None: + if ret == None: print ("\nCould not find id number for '%s'" % filepath) return None nodeid = ret[0] @@ -116,7 +112,7 @@ class Graph(object): m, s = sec / 60, sec % 60 print ("\n------ Summary for %s ------\ \nTotal Build Time (mm:ss) = %d:%d\nNum Downstream Commands = %d" - % (f, m, s, node.num_cmds)) + % (f, m, s, node.num_cmds)) def populate(self): # make nodes for files with downstream commands @@ -131,4 +127,5 @@ class Graph(object): def get_cost_dict(self): if self.results is None: self.populate() - return {k: v for k, v in self.results if v > 0} + return {k:v for k,v in self.results if v > 0} + diff --git a/python/mozbuild/mozbuild/analyze/hg.py b/python/mozbuild/mozbuild/analyze/hg.py index f7c4623bd197..35a626669a55 100644 --- a/python/mozbuild/mozbuild/analyze/hg.py +++ b/python/mozbuild/mozbuild/analyze/hg.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import bisect import gzip import json @@ -19,36 +17,31 @@ PUSHLOG_CHUNK_SIZE = 500 URL = 'https://hg.mozilla.org/mozilla-central/json-pushes?' - def unix_epoch(date): - return (date - datetime(1970, 1, 1)).total_seconds() - + return (date - datetime(1970,1,1)).total_seconds() def unix_from_date(n, today): return unix_epoch(today - timedelta(days=n)) - def get_lastpid(session): return session.get(URL+'&version=2').json()['lastpushid'] - def get_pushlog_chunk(session, start, end): # returns pushes sorted by date res = session.get(URL+'version=1&startID={0}&\ endID={1}&full=1'.format(start, end)).json() - return sorted(res.items(), key=lambda x: x[1]['date']) - + return sorted(res.items(), key = lambda x: x[1]['date']) def collect_data(session, date): - if date < 1206031764: # first push - raise Exception("No pushes exist before March 20, 2008.") + if date < 1206031764: #first push + raise Exception ("No pushes exist before March 20, 2008.") lastpushid = get_lastpid(session) data = [] start_id = lastpushid - PUSHLOG_CHUNK_SIZE end_id = lastpushid + 1 while True: res = get_pushlog_chunk(session, start_id, end_id) - starting_date = res[0][1]['date'] # date of oldest push in chunk + starting_date = res[0][1]['date'] # date of oldest push in chunk dates = [x[1]['date'] for x in res] if starting_date < date: i = bisect.bisect_left(dates, date) @@ -59,12 +52,10 @@ def collect_data(session, date): end_id = start_id + 1 start_id = start_id - PUSHLOG_CHUNK_SIZE - def get_data(epoch): session = requests.Session() data = collect_data(session, epoch) - return {k: v for sublist in data for (k, v) in sublist} - + return {k:v for sublist in data for (k,v) in sublist} class Pushlog(object): @@ -87,7 +78,6 @@ class Pushlog(object): keys.sort() return keys - class Push(object): def __init__(self, pid, p_dict): @@ -95,7 +85,6 @@ class Push(object): self.date = p_dict['date'] self.files = [f for x in p_dict['changesets'] for f in x['files']] - class Report(object): def __init__(self, days, path=None, cost_dict=None): @@ -123,7 +112,7 @@ class Report(object): cost = costs.get(f) count = counts.get(f) if cost is not None: - res.append((f, cost, count, round(cost*count, 3))) + res.append((f, cost, count, round(cost*count,3))) return res def get_sorted_report(self, format): @@ -154,8 +143,7 @@ class Report(object): res = self.get_sorted_report(format) if limit is not None: res = self.cut(limit, res) - for x in res: - data.append(x) + for x in res: data.append(x) if format == 'pretty': print (data) else: @@ -172,3 +160,4 @@ class Report(object): with open(file_path, 'wb') as f: f.write(content) print ("Created report: %s" % file_path) + diff --git a/python/mozbuild/mozbuild/android_version_code.py b/python/mozbuild/mozbuild/android_version_code.py index 8eed2ef5d769..ff9dc6c6b155 100644 --- a/python/mozbuild/mozbuild/android_version_code.py +++ b/python/mozbuild/mozbuild/android_version_code.py @@ -11,8 +11,7 @@ import time # Builds before this build ID use the v0 version scheme. Builds after this # build ID use the v1 version scheme. -V1_CUTOFF = 20150801000000 # YYYYmmddHHMMSS - +V1_CUTOFF = 20150801000000 # YYYYmmddHHMMSS def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): base = int(str(buildid)[:10]) @@ -31,7 +30,6 @@ def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): raise ValueError("Don't know how to compute android:versionCode " "for CPU arch %s" % cpu_arch) - def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): '''Generate a v1 android:versionCode. @@ -136,7 +134,6 @@ def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): return version - def android_version_code(buildid, *args, **kwargs): base = int(str(buildid)) if base < V1_CUTOFF: @@ -169,9 +166,9 @@ def main(argv): args = parser.parse_args(argv) code = android_version_code(args.buildid, - cpu_arch=args.cpu_arch, - min_sdk=args.min_sdk, - max_sdk=args.max_sdk) + cpu_arch=args.cpu_arch, + min_sdk=args.min_sdk, + max_sdk=args.max_sdk) print(code) return 0 diff --git a/python/mozbuild/mozbuild/artifact_builds.py b/python/mozbuild/mozbuild/artifact_builds.py index 5bf4cefb7b9a..04e108cecdcb 100644 --- a/python/mozbuild/mozbuild/artifact_builds.py +++ b/python/mozbuild/mozbuild/artifact_builds.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - # The values correspond to entries at # https://tools.taskcluster.net/index/artifacts/#gecko.v2.mozilla-central.latest/gecko.v2.mozilla-central.latest JOB_CHOICES = { diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py index a4211e9406ba..47e9451ec57f 100644 --- a/python/mozbuild/mozbuild/artifacts.py +++ b/python/mozbuild/mozbuild/artifacts.py @@ -77,8 +77,7 @@ from mozpack.mozjar import ( from mozpack.packager.unpack import UnpackFinder import mozpack.path as mozpath -# Number of candidate pushheads to cache per parent changeset. -NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50 +NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50 # Number of candidate pushheads to cache per parent changeset. # Number of parent changesets to consider as possible pushheads. # There isn't really such a thing as a reasonable default here, because we don't @@ -314,8 +313,8 @@ class AndroidArtifactJob(ArtifactJob): dirname, basename = os.path.split(p) self.log(logging.INFO, 'artifact', - {'basename': basename}, - 'Adding {basename} to processed archive') + {'basename': basename}, + 'Adding {basename} to processed archive') basedir = 'bin' if not basename.endswith('.so'): @@ -324,8 +323,7 @@ class AndroidArtifactJob(ArtifactJob): writer.add(basename.encode('utf-8'), f.open()) def process_symbols_archive(self, filename, processed_filename): - ArtifactJob.process_symbols_archive( - self, filename, processed_filename, skip_compressed=True) + ArtifactJob.process_symbols_archive(self, filename, processed_filename, skip_compressed=True) if self._symbols_archive_suffix != 'crashreporter-symbols-full.zip': return @@ -338,11 +336,9 @@ class AndroidArtifactJob(ArtifactJob): if not filename.endswith('.gz'): continue - # Uncompress "libxul.so/D3271457813E976AE7BF5DAFBABABBFD0/libxul.so.dbg.gz" - # into "libxul.so.dbg". + # Uncompress "libxul.so/D3271457813E976AE7BF5DAFBABABBFD0/libxul.so.dbg.gz" into "libxul.so.dbg". # - # After running `settings append target.debug-file-search-paths $file`, - # where file=/path/to/topobjdir/dist/crashreporter-symbols, + # After `settings append target.debug-file-search-paths /path/to/topobjdir/dist/crashreporter-symbols`, # Android Studio's lldb (7.0.0, at least) will find the ELF debug symbol files. # # There are other paths that will work but none seem more desireable. See @@ -351,8 +347,7 @@ class AndroidArtifactJob(ArtifactJob): destpath = mozpath.join('crashreporter-symbols', basename) self.log(logging.INFO, 'artifact', {'destpath': destpath}, - 'Adding uncompressed ELF debug symbol file ' - '{destpath} to processed archive') + 'Adding uncompressed ELF debug symbol file {destpath} to processed archive') writer.add(destpath.encode('utf-8'), gzip.GzipFile(fileobj=reader[filename].uncompressed_data)) @@ -445,8 +440,8 @@ class MacArtifactJob(ArtifactJob): oldcwd = os.getcwd() try: self.log(logging.INFO, 'artifact', - {'tempdir': tempdir}, - 'Unpacking DMG into {tempdir}') + {'tempdir': tempdir}, + 'Unpacking DMG into {tempdir}') if self._substs['HOST_OS_ARCH'] == 'Linux': # This is a cross build, use hfsplus and dmg tools to extract the dmg. os.chdir(tempdir) @@ -491,8 +486,8 @@ class MacArtifactJob(ArtifactJob): for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', - {'path': p}, - 'Adding {path} to processed archive') + {'path': p}, + 'Adding {path} to processed archive') destpath = mozpath.join('bin', os.path.basename(p)) writer.add(destpath.encode('utf-8'), f, mode=f.mode) @@ -512,8 +507,8 @@ class MacArtifactJob(ArtifactJob): shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, 'artifact', - {'tempdir': tempdir}, - 'Unable to delete {tempdir}') + {'tempdir': tempdir}, + 'Unable to delete {tempdir}') pass @@ -566,8 +561,8 @@ class WinArtifactJob(ArtifactJob): basename = mozpath.relpath(p, self.product) basename = mozpath.join('bin', basename) self.log(logging.INFO, 'artifact', - {'basename': basename}, - 'Adding {basename} to processed archive') + {'basename': basename}, + 'Adding {basename} to processed archive') writer.add(basename.encode('utf-8'), f.open(), mode=f.mode) added_entry = True @@ -659,8 +654,7 @@ class CacheManager(object): Provide simple logging. ''' - def __init__(self, cache_dir, cache_name, cache_size, cache_callback=None, - log=None, skip_cache=False): + def __init__(self, cache_dir, cache_name, cache_size, cache_callback=None, log=None, skip_cache=False): self._skip_cache = skip_cache self._cache = pylru.lrucache(cache_size, callback=cache_callback) self._cache_filename = mozpath.join(cache_dir, cache_name + '-cache.pickle') @@ -674,8 +668,8 @@ class CacheManager(object): def load_cache(self): if self._skip_cache: self.log(logging.INFO, 'artifact', - {}, - 'Skipping cache: ignoring load_cache!') + {}, + 'Skipping cache: ignoring load_cache!') return try: @@ -687,26 +681,25 @@ class CacheManager(object): # exceptions, so it's not worth trying to be fine grained here. # We ignore any exception, so the cache is effectively dropped. self.log(logging.INFO, 'artifact', - {'filename': self._cache_filename, 'exception': repr(e)}, - 'Ignoring exception unpickling cache file {filename}: {exception}') + {'filename': self._cache_filename, 'exception': repr(e)}, + 'Ignoring exception unpickling cache file {filename}: {exception}') pass def dump_cache(self): if self._skip_cache: self.log(logging.INFO, 'artifact', - {}, - 'Skipping cache: ignoring dump_cache!') + {}, + 'Skipping cache: ignoring dump_cache!') return ensureParentDir(self._cache_filename) - pickle.dump(list(reversed(list(self._cache.items()))), - open(self._cache_filename, 'wb'), -1) + pickle.dump(list(reversed(list(self._cache.items()))), open(self._cache_filename, 'wb'), -1) def clear_cache(self): if self._skip_cache: self.log(logging.INFO, 'artifact', - {}, - 'Skipping cache: ignoring clear_cache!') + {}, + 'Skipping cache: ignoring clear_cache!') return with self: @@ -719,13 +712,11 @@ class CacheManager(object): def __exit__(self, type, value, traceback): self.dump_cache() - class PushheadCache(CacheManager): '''Helps map tree/revision pairs to parent pushheads according to the pushlog.''' def __init__(self, cache_dir, log=None, skip_cache=False): - CacheManager.__init__(self, cache_dir, 'pushhead_cache', - MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) + CacheManager.__init__(self, cache_dir, 'pushhead_cache', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) @cachedmethod(operator.attrgetter('_cache')) def parent_pushhead_id(self, tree, revision): @@ -752,13 +743,11 @@ class PushheadCache(CacheManager): p['changesets'][-1] for p in result['pushes'].values() ] - class TaskCache(CacheManager): '''Map candidate pushheads to Task Cluster task IDs and artifact URLs.''' def __init__(self, cache_dir, log=None, skip_cache=False): - CacheManager.__init__(self, cache_dir, 'artifact_url', - MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) + CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) @cachedmethod(operator.attrgetter('_cache')) def artifacts(self, tree, job, artifact_job_class, rev): @@ -793,8 +782,7 @@ class TaskCache(CacheManager): except KeyError: # Not all revisions correspond to pushes that produce the job we # care about; and even those that do may not have completed yet. - raise ValueError( - 'Task for {namespace} does not exist (yet)!'.format(namespace=namespace)) + raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace)) return taskId, list_artifacts(taskId) @@ -832,15 +820,13 @@ class Artifacts(object): substs=self._substs) except KeyError: self.log(logging.INFO, 'artifact', - {'job': self._job}, - 'Unknown job {job}') + {'job': self._job}, + 'Unknown job {job}') raise KeyError("Unknown job") self._task_cache = TaskCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) - self._artifact_cache = ArtifactCache( - self._cache_dir, log=self._log, skip_cache=self._skip_cache) - self._pushhead_cache = PushheadCache( - self._cache_dir, log=self._log, skip_cache=self._skip_cache) + self._artifact_cache = ArtifactCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) + self._pushhead_cache = PushheadCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) def log(self, *args, **kwargs): if self._log: @@ -964,12 +950,11 @@ class Artifacts(object): There are no public revisions. This can happen if the repository is created from bundle file and never pulled from remote. Please run `hg pull` and build again. -see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles\ -""") +see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code/Mercurial/Bundles""") self.log(logging.INFO, 'artifact', - {'len': len(last_revs)}, - 'hg suggested {len} candidate revisions') + {'len': len(last_revs)}, + 'hg suggested {len} candidate revisions') def to_pair(line): rev, node = line.split(':', 1) @@ -1010,16 +995,14 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code yield candidate_pushheads[rev], rev if not count: - raise Exception( - 'Could not find any candidate pushheads in the last {num} revisions.\n' - 'Search started with {rev}, which must be known to Mozilla automation.\n\n' - 'see https://developer.mozilla.org/en-US/docs/Artifact_builds'.format( - rev=last_revs[0], num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT)) + raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n' + 'Search started with {rev}, which must be known to Mozilla automation.\n\n' + 'see https://developer.mozilla.org/en-US/docs/Artifact_builds'.format( + rev=last_revs[0], num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT)) def find_pushhead_artifacts(self, task_cache, job, tree, pushhead): try: - taskId, artifacts = task_cache.artifacts( - tree, job, self._artifact_job.__class__, pushhead) + taskId, artifacts = task_cache.artifacts(tree, job, self._artifact_job.__class__, pushhead) except ValueError: return None @@ -1040,32 +1023,32 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code def install_from_file(self, filename, distdir): self.log(logging.INFO, 'artifact', - {'filename': filename}, - 'Installing from {filename}') + {'filename': filename}, + 'Installing from {filename}') # Do we need to post-process? processed_filename = filename + PROCESSED_SUFFIX if self._skip_cache and os.path.exists(processed_filename): self.log(logging.INFO, 'artifact', - {'path': processed_filename}, - 'Skipping cache: removing cached processed artifact {path}') + {'path': processed_filename}, + 'Skipping cache: removing cached processed artifact {path}') os.remove(processed_filename) if not os.path.exists(processed_filename): self.log(logging.INFO, 'artifact', - {'filename': filename}, - 'Processing contents of {filename}') + {'filename': filename}, + 'Processing contents of {filename}') self.log(logging.INFO, 'artifact', - {'processed_filename': processed_filename}, - 'Writing processed {processed_filename}') + {'processed_filename': processed_filename}, + 'Writing processed {processed_filename}') self._artifact_job.process_artifact(filename, processed_filename) self._artifact_cache._persist_limit.register_file(processed_filename) self.log(logging.INFO, 'artifact', - {'processed_filename': processed_filename}, - 'Installing from processed {processed_filename}') + {'processed_filename': processed_filename}, + 'Installing from processed {processed_filename}') # Copy all .so files, avoiding modification where possible. ensureParentDir(mozpath.join(distdir, '.dummy')) @@ -1079,21 +1062,20 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code shutil.copyfileobj(zf.open(info), fh) file_existed, file_updated = fh.close() self.log(logging.INFO, 'artifact', - {'updating': 'Updating' if file_updated else 'Not updating', - 'filename': n}, - '{updating} {filename}') + {'updating': 'Updating' if file_updated else 'Not updating', 'filename': n}, + '{updating} {filename}') if not file_existed or file_updated: # Libraries and binaries may need to be marked executable, # depending on platform. - perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689. - perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r. + perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689. + perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r. os.chmod(n, perms) return 0 def install_from_url(self, url, distdir): self.log(logging.INFO, 'artifact', - {'url': url}, - 'Installing from {url}') + {'url': url}, + 'Installing from {url}') filename = self._artifact_cache.fetch(url) return self.install_from_file(filename, distdir) @@ -1135,7 +1117,7 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code try: if self._hg: revision = subprocess.check_output([self._hg, 'log', '--template', '{node}\n', - '-r', revset], cwd=self._topsrcdir).strip() + '-r', revset], cwd=self._topsrcdir).strip() elif self._git: revset = subprocess.check_output([ self._git, 'rev-parse', '%s^{commit}' % revset], @@ -1183,8 +1165,7 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code url = get_artifact_url(taskId, artifact_name) urls.append(url) if not urls: - raise ValueError( - 'Task {taskId} existed, but no artifacts found!'.format(taskId=taskId)) + raise ValueError('Task {taskId} existed, but no artifacts found!'.format(taskId=taskId)) for url in urls: if self.install_from_url(url, distdir): return 1 @@ -1213,10 +1194,11 @@ see https://developer.mozilla.org/en-US/docs/Mozilla/Developer_guide/Source_Code return self.install_from_recent(distdir) + def clear_cache(self): self.log(logging.INFO, 'artifact', - {}, - 'Deleting cached artifacts and caches.') + {}, + 'Deleting cached artifacts and caches.') self._task_cache.clear_cache() self._artifact_cache.clear_cache() self._pushhead_cache.clear_cache() diff --git a/python/mozbuild/mozbuild/backend/__init__.py b/python/mozbuild/mozbuild/backend/__init__.py index 6b25793aba59..8c8979db0787 100644 --- a/python/mozbuild/mozbuild/backend/__init__.py +++ b/python/mozbuild/mozbuild/backend/__init__.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - backends = { 'ChromeMap': 'mozbuild.codecoverage.chrome_map', 'CompileDB': 'mozbuild.compilation.database', diff --git a/python/mozbuild/mozbuild/backend/base.py b/python/mozbuild/mozbuild/backend/base.py index d61be09e4864..14023e45cb5a 100644 --- a/python/mozbuild/mozbuild/backend/base.py +++ b/python/mozbuild/mozbuild/backend/base.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals from abc import ( ABCMeta, diff --git a/python/mozbuild/mozbuild/backend/cargo_build_defs.py b/python/mozbuild/mozbuild/backend/cargo_build_defs.py index 49451c30307b..298759830639 100644 --- a/python/mozbuild/mozbuild/backend/cargo_build_defs.py +++ b/python/mozbuild/mozbuild/backend/cargo_build_defs.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals cargo_extra_outputs = { 'bindgen': [ diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index 7b7c6325b70c..dcfb6c1cee49 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import itertools import json @@ -33,6 +33,7 @@ from mozbuild.frontend.data import ( GnProjectData, HostLibrary, HostGeneratedSources, + HostRustLibrary, IPDLCollection, LocalizedPreprocessedFiles, LocalizedFiles, @@ -52,10 +53,10 @@ from mozbuild.preprocessor import Preprocessor from mozpack.chrome.manifest import parse_manifest_line from mozbuild.util import ( + group_unified_files, mkdir, ) - class XPIDLManager(object): """Helps manage XPCOM IDLs in the context of the build system.""" @@ -101,7 +102,6 @@ class XPIDLManager(object): """ return itertools.chain(*[m.stems() for m in self.modules.itervalues()]) - class BinariesCollection(object): """Tracks state of binaries produced by the build.""" @@ -109,7 +109,6 @@ class BinariesCollection(object): self.shared_libraries = [] self.programs = [] - class CommonBackend(BuildBackend): """Holds logic common to all build backends.""" @@ -183,8 +182,7 @@ class CommonBackend(BuildBackend): return False elif isinstance(obj, Exports): - objdir_files = [f.full_path for path, files in obj.files.walk() - for f in files if isinstance(f, ObjDirPath)] + objdir_files = [f.full_path for path, files in obj.files.walk() for f in files if isinstance(f, ObjDirPath)] if objdir_files: self._handle_generated_sources(objdir_files) return False @@ -203,9 +201,9 @@ class CommonBackend(BuildBackend): if len(self._idl_manager.modules): self._write_rust_xpidl_summary(self._idl_manager) self._handle_idl_manager(self._idl_manager) - self._handle_generated_sources( - mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % stem) - for stem in self._idl_manager.idl_stems()) + self._handle_generated_sources(mozpath.join(self.environment.topobjdir, 'dist/include/%s.h' % stem) + for stem in self._idl_manager.idl_stems()) + for config in self._configs: self.backend_input_files.add(config.source) @@ -296,7 +294,7 @@ class CommonBackend(BuildBackend): seen_libs.add(lib) os_libs.append(lib) - return (objs, sorted(seen_pgo_gen_only_objs), no_pgo_objs, + return (objs, sorted(seen_pgo_gen_only_objs), no_pgo_objs, \ shared_libs, os_libs, static_libs) def _make_list_file(self, kind, objdir, objs, name): @@ -334,8 +332,7 @@ class CommonBackend(BuildBackend): return ref def _handle_generated_sources(self, files): - self._generated_sources.update(mozpath.relpath( - f, self.environment.topobjdir) for f in files) + self._generated_sources.update(mozpath.relpath(f, self.environment.topobjdir) for f in files) def _handle_webidl_collection(self, webidls): @@ -399,7 +396,7 @@ class CommonBackend(BuildBackend): includeTemplate += ( '\n' '#if defined(_WINDOWS_) && !defined(MOZ_WRAPPED_WINDOWS_H)\n' - '#pragma message("wrapper failure reason: " MOZ_WINDOWS_WRAPPER_DISABLED_REASON)\n' # noqa + '#pragma message("wrapper failure reason: " MOZ_WINDOWS_WRAPPER_DISABLED_REASON)\n' '#error "%(cppfile)s included unwrapped windows.h"\n' "#endif") includeTemplate += ( @@ -414,7 +411,7 @@ class CommonBackend(BuildBackend): 'so it cannot be built in unified mode."\n' '#undef INITGUID\n' '#endif') - f.write('\n'.join(includeTemplate % {"cppfile": s} for + f.write('\n'.join(includeTemplate % { "cppfile": s } for s in source_filenames)) def _write_unified_files(self, unified_source_mapping, output_directory, diff --git a/python/mozbuild/mozbuild/backend/configenvironment.py b/python/mozbuild/mozbuild/backend/configenvironment.py index f678f41f5731..76eec5889e42 100644 --- a/python/mozbuild/mozbuild/backend/configenvironment.py +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import sys @@ -49,7 +49,7 @@ class BuildConfig(object): # cache the compiled code as it can be reused # we cache it the first time, or if the file changed - if path not in code_cache or code_cache[path][0] != mtime: + if not path in code_cache or code_cache[path][0] != mtime: # Add config.status manually to sys.modules so it gets picked up by # iter_modules_in_path() for automatic dependencies. mod = ModuleType('config.status') @@ -118,7 +118,7 @@ class ConfigEnvironment(object): """ def __init__(self, topsrcdir, topobjdir, defines=None, - non_global_defines=None, substs=None, source=None, mozconfig=None): + non_global_defines=None, substs=None, source=None, mozconfig=None): if not source: source = mozpath.join(topobjdir, 'config.status') @@ -148,35 +148,20 @@ class ConfigEnvironment(object): self.bin_suffix = self.substs.get('BIN_SUFFIX', '') global_defines = [name for name in self.defines - if name not in self.non_global_defines] - self.substs["ACDEFINES"] = ' '.join( - [ - '-D%s=%s' % (name, shell_quote(self.defines[name]).replace('$', '$$')) - for name in sorted(global_defines) - ] - ) - + if not name in self.non_global_defines] + self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name, + shell_quote(self.defines[name]).replace('$', '$$')) + for name in sorted(global_defines)]) def serialize(name, obj): if isinstance(obj, StringTypes): return obj if isinstance(obj, Iterable): return ' '.join(obj) raise Exception('Unhandled type %s for %s', type(obj), str(name)) - self.substs['ALLSUBSTS'] = '\n'.join( - sorted([ - '%s = %s' % ( - name, - serialize(name, self.substs[name]) - ) - for name in self.substs if self.substs[name] - ]) - ) - self.substs['ALLEMPTYSUBSTS'] = '\n'.join( - sorted([ - '%s =' % name - for name in self.substs if not self.substs[name] - ]) - ) + self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name, + serialize(name, self.substs[name])) for name in self.substs if self.substs[name]])) + self.substs['ALLEMPTYSUBSTS'] = '\n'.join(sorted(['%s =' % name + for name in self.substs if not self.substs[name]])) self.substs = ReadOnlyDict(self.substs) @@ -229,7 +214,7 @@ class ConfigEnvironment(object): config = BuildConfig.from_config_status(path) return ConfigEnvironment(config.topsrcdir, config.topobjdir, - config.defines, config.non_global_defines, config.substs, path) + config.defines, config.non_global_defines, config.substs, path) class PartialConfigDict(object): @@ -239,7 +224,6 @@ class PartialConfigDict(object): similar for substs), where the value of FOO is delay-loaded until it is needed. """ - def __init__(self, config_statusd, typ, environ_override=False): self._dict = {} self._datadir = mozpath.join(config_statusd, typ) @@ -354,7 +338,6 @@ class PartialConfigEnvironment(object): intended to be used instead of the defines structure from config.status so that scripts can depend directly on its value. """ - def __init__(self, topobjdir): config_statusd = mozpath.join(topobjdir, 'config.statusd') self.substs = PartialConfigDict(config_statusd, 'substs', environ_override=True) @@ -370,8 +353,8 @@ class PartialConfigEnvironment(object): if name not in config['non_global_defines'] ] acdefines = ' '.join(['-D%s=%s' % (name, - shell_quote(config['defines'][name]).replace('$', '$$')) - for name in sorted(global_defines)]) + shell_quote(config['defines'][name]).replace('$', '$$')) + for name in sorted(global_defines)]) substs['ACDEFINES'] = acdefines all_defines = OrderedDict() diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/mozbuild/backend/cpp_eclipse.py index b91eee661907..be8daf5ac9b1 100644 --- a/python/mozbuild/mozbuild/backend/cpp_eclipse.py +++ b/python/mozbuild/mozbuild/backend/cpp_eclipse.py @@ -2,18 +2,22 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import errno import glob +import random import os import shutil import subprocess +import types from xml.sax.saxutils import quoteattr +import xml.etree.ElementTree as ET from .common import CommonBackend from ..frontend.data import ( ComputedFlags, + Defines, ) from mozbuild.base import ExecutionSummary @@ -22,7 +26,6 @@ from mozbuild.base import ExecutionSummary # Open eclipse: # /Users/bgirard/mozilla/eclipse/eclipse/eclipse/eclipse -data $PWD/workspace - class CppEclipseBackend(CommonBackend): """Backend that generates Cpp Eclipse project files. """ @@ -78,8 +81,7 @@ class CppEclipseBackend(CommonBackend): # Note that unlike VS, Eclipse' indexer seem to crawl the headers and # isn't picky about the local includes. if isinstance(obj, ComputedFlags): - args = self._args_for_dirs.setdefault( - 'tree/' + reldir, {'includes': [], 'defines': []}) + args = self._args_for_dirs.setdefault('tree/' + reldir, {'includes': [], 'defines': []}) # use the same args for any objdirs we include: if reldir == 'dom/bindings': self._args_for_dirs.setdefault('generated-webidl', args) @@ -103,8 +105,7 @@ class CppEclipseBackend(CommonBackend): def consume_finished(self): settings_dir = os.path.join(self._project_dir, '.settings') launch_dir = os.path.join(self._project_dir, 'RunConfigurations') - workspace_settings_dir = os.path.join( - self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings') + workspace_settings_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings') for dir_name in [self._project_dir, settings_dir, launch_dir, workspace_settings_dir, self._workspace_lang_dir]: try: @@ -128,25 +129,22 @@ class CppEclipseBackend(CommonBackend): workspace_language_path = os.path.join(self._workspace_lang_dir, 'language.settings.xml') with open(workspace_language_path, 'wb') as fh: workspace_lang_settings = WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE - workspace_lang_settings = workspace_lang_settings.replace( - "@COMPILER_FLAGS@", self._cxx + " " + self._cppflags) + workspace_lang_settings = workspace_lang_settings.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags); fh.write(workspace_lang_settings) self._write_launch_files(launch_dir) - core_resources_prefs_path = os.path.join( - workspace_settings_dir, 'org.eclipse.core.resources.prefs') + core_resources_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.core.resources.prefs') with open(core_resources_prefs_path, 'wb') as fh: - fh.write(STATIC_CORE_RESOURCES_PREFS) + fh.write(STATIC_CORE_RESOURCES_PREFS); - core_runtime_prefs_path = os.path.join( - workspace_settings_dir, 'org.eclipse.core.runtime.prefs') + core_runtime_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.core.runtime.prefs') with open(core_runtime_prefs_path, 'wb') as fh: - fh.write(STATIC_CORE_RUNTIME_PREFS) + fh.write(STATIC_CORE_RUNTIME_PREFS); ui_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.ui.prefs') with open(ui_prefs_path, 'wb') as fh: - fh.write(STATIC_UI_PREFS) + fh.write(STATIC_UI_PREFS); cdt_ui_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.cdt.ui.prefs') cdt_ui_prefs = STATIC_CDT_UI_PREFS @@ -157,11 +155,10 @@ class CppEclipseBackend(CommonBackend): XML_PREF_TEMPLATE = """\\n""" for line in FORMATTER_SETTINGS.splitlines(): [pref, val] = line.split("=") - cdt_ui_prefs += XML_PREF_TEMPLATE.replace("@PREF_NAME@", - pref).replace("@PREF_VAL@", val) + cdt_ui_prefs += XML_PREF_TEMPLATE.replace("@PREF_NAME@", pref).replace("@PREF_VAL@", val) cdt_ui_prefs += "\\n\\n" with open(cdt_ui_prefs_path, 'wb') as fh: - fh.write(cdt_ui_prefs) + fh.write(cdt_ui_prefs); cdt_core_prefs_path = os.path.join(workspace_settings_dir, 'org.eclipse.cdt.core.prefs') with open(cdt_core_prefs_path, 'wb') as fh: @@ -171,11 +168,11 @@ class CppEclipseBackend(CommonBackend): # as the active formatter all its prefs are set in this prefs file, # so we need add those now: cdt_core_prefs += FORMATTER_SETTINGS - fh.write(cdt_core_prefs) + fh.write(cdt_core_prefs); - editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs") + editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs"); with open(editor_prefs_path, 'wb') as fh: - fh.write(EDITOR_SETTINGS) + fh.write(EDITOR_SETTINGS); # Now import the project into the workspace self._import_project() @@ -191,7 +188,7 @@ class CppEclipseBackend(CommonBackend): self._write_noindex() try: - subprocess.check_call( + process = subprocess.check_call( ["eclipse", "-application", "-nosplash", "org.eclipse.cdt.managedbuilder.core.headlessbuild", "-data", self._workspace_dir, "-importAll", self._project_dir]) @@ -211,7 +208,7 @@ class CppEclipseBackend(CommonBackend): def _write_noindex(self): noindex_path = os.path.join(self._project_dir, '.settings/org.eclipse.cdt.core.prefs') with open(noindex_path, 'wb') as fh: - fh.write(NOINDEX_TEMPLATE) + fh.write(NOINDEX_TEMPLATE); def _remove_noindex(self): # Below we remove the config file that temporarily disabled the indexer @@ -260,8 +257,7 @@ class CppEclipseBackend(CommonBackend): dirsettings_template = LANGUAGE_SETTINGS_TEMPLATE_DIR_HEADER # Add OS_COMPILE_CXXFLAGS args (same as OS_COMPILE_CFLAGS): - dirsettings_template = dirsettings_template.replace('@PREINCLUDE_FILE_PATH@', os.path.join( - self.environment.topobjdir, 'dist/include/mozilla-config.h')) + dirsettings_template = dirsettings_template.replace('@PREINCLUDE_FILE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include/mozilla-config.h')) dirsettings_template += add_define('MOZILLA_CLIENT', '1') # Add EXTRA_INCLUDES args: @@ -308,7 +304,7 @@ class CppEclipseBackend(CommonBackend): # netwerk/sctp/src uses -U__APPLE__ on Mac # XXX We should make this code smart enough to remove existing defines. continue - d = d[2:] # get rid of leading "-D" + d = d[2:] # get rid of leading "-D" name_value = d.split("=", 1) name = name_value[0] value = "" @@ -318,8 +314,7 @@ class CppEclipseBackend(CommonBackend): dirsettings += LANGUAGE_SETTINGS_TEMPLATE_DIR_FOOTER fh.write(dirsettings) - fh.write(LANGUAGE_SETTINGS_TEMPLATE_FOOTER.replace( - "@COMPILER_FLAGS@", self._cxx + " " + self._cppflags)) + fh.write(LANGUAGE_SETTINGS_TEMPLATE_FOOTER.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags)) def _write_launch_files(self, launch_dir): bin_dir = os.path.join(self.environment.topobjdir, 'dist') @@ -339,25 +334,21 @@ class CppEclipseBackend(CommonBackend): launch = launch.replace('@LAUNCH_ARGS@', '-P -no-remote') fh.write(launch) - # TODO Add more launch configs (and delegate calls to mach) + #TODO Add more launch configs (and delegate calls to mach) def _write_project(self, fh): - project = PROJECT_TEMPLATE + project = PROJECT_TEMPLATE; project = project.replace('@PROJECT_NAME@', self._project_name) project = project.replace('@PROJECT_TOPSRCDIR@', self.environment.topsrcdir) - project = project.replace('@GENERATED_IPDL_FILES@', os.path.join( - self.environment.topobjdir, "ipc", "ipdl")) - project = project.replace('@GENERATED_WEBIDL_FILES@', os.path.join( - self.environment.topobjdir, "dom", "bindings")) + project = project.replace('@GENERATED_IPDL_FILES@', os.path.join(self.environment.topobjdir, "ipc", "ipdl")) + project = project.replace('@GENERATED_WEBIDL_FILES@', os.path.join(self.environment.topobjdir, "dom", "bindings")) fh.write(project) def _write_cproject(self, fh): cproject_header = CPROJECT_TEMPLATE_HEADER - cproject_header = cproject_header.replace( - '@PROJECT_TOPSRCDIR@', self.environment.topobjdir) - cproject_header = cproject_header.replace( - '@MACH_COMMAND@', os.path.join(self.environment.topsrcdir, 'mach')) + cproject_header = cproject_header.replace('@PROJECT_TOPSRCDIR@', self.environment.topobjdir) + cproject_header = cproject_header.replace('@MACH_COMMAND@', os.path.join(self.environment.topsrcdir, 'mach')) fh.write(cproject_header) fh.write(CPROJECT_TEMPLATE_FOOTER) @@ -378,8 +369,8 @@ PROJECT_TEMPLATE = """ org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - - + + @@ -477,7 +468,7 @@ CPROJECT_TEMPLATE_HEADER = """ - + """ @@ -491,9 +482,9 @@ CPROJECT_TEMPLATE_FILEINFO = """ """ CPROJECT_TEMPLATE_FOOTER = """ - - - + + + @@ -535,38 +526,38 @@ WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE = """ - - - - + + + + """ -LANGUAGE_SETTINGS_TEMPLATE_DIR_HEADER = """ - - - +LANGUAGE_SETTINGS_TEMPLATE_DIR_HEADER = """ + + + """ -LANGUAGE_SETTINGS_TEMPLATE_DIR_INCLUDE = """ - - +LANGUAGE_SETTINGS_TEMPLATE_DIR_INCLUDE = """ + + """ -LANGUAGE_SETTINGS_TEMPLATE_DIR_DEFINE = """ +LANGUAGE_SETTINGS_TEMPLATE_DIR_DEFINE = """ """ -LANGUAGE_SETTINGS_TEMPLATE_DIR_FOOTER = """ +LANGUAGE_SETTINGS_TEMPLATE_DIR_FOOTER = """ """ -LANGUAGE_SETTINGS_TEMPLATE_FOOTER = """ - - - - - - - - +LANGUAGE_SETTINGS_TEMPLATE_FOOTER = """ + + + + + + + + """ @@ -624,21 +615,21 @@ undoHistorySize=200 """ -STATIC_CORE_RESOURCES_PREFS = """eclipse.preferences.version=1 +STATIC_CORE_RESOURCES_PREFS="""eclipse.preferences.version=1 refresh.enabled=true """ -STATIC_CORE_RUNTIME_PREFS = """eclipse.preferences.version=1 +STATIC_CORE_RUNTIME_PREFS="""eclipse.preferences.version=1 content-types/org.eclipse.cdt.core.cxxSource/file-extensions=mm content-types/org.eclipse.core.runtime.xml/file-extensions=xul content-types/org.eclipse.wst.jsdt.core.jsSource/file-extensions=jsm """ -STATIC_UI_PREFS = """eclipse.preferences.version=1 +STATIC_UI_PREFS="""eclipse.preferences.version=1 showIntro=false """ -STATIC_CDT_CORE_PREFS = """eclipse.preferences.version=1 +STATIC_CDT_CORE_PREFS="""eclipse.preferences.version=1 indexer.updatePolicy=0 """ @@ -806,7 +797,7 @@ org.eclipse.cdt.core.formatter.tabulation.size=2 org.eclipse.cdt.core.formatter.use_tabs_only_for_leading_indentations=false """ -STATIC_CDT_UI_PREFS = """eclipse.preferences.version=1 +STATIC_CDT_UI_PREFS="""eclipse.preferences.version=1 buildConsoleLines=10000 Console.limitConsoleOutput=false ensureNewlineAtEOF=false diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py index e8da785d459c..61479a46514f 100644 --- a/python/mozbuild/mozbuild/backend/fastermake.py +++ b/python/mozbuild/mozbuild/backend/fastermake.py @@ -134,12 +134,10 @@ class FasterMakeBackend(CommonBackend, PartialBackend): elif isinstance(obj, GeneratedFile): if obj.outputs: - first_output = mozpath.relpath(mozpath.join( - obj.objdir, obj.outputs[0]), self.environment.topobjdir) + first_output = mozpath.relpath(mozpath.join(obj.objdir, obj.outputs[0]), self.environment.topobjdir) for o in obj.outputs[1:]: fullpath = mozpath.join(obj.objdir, o) - self._generated_files_map[mozpath.relpath( - fullpath, self.environment.topobjdir)] = first_output + self._generated_files_map[mozpath.relpath(fullpath, self.environment.topobjdir)] = first_output # We don't actually handle GeneratedFiles, we just need to know if # we can build multiple of them from a single make invocation in the # faster backend. @@ -196,6 +194,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): mk.create_rule([target]).add_dependencies( '$(TOPOBJDIR)/%s' % d for d in deps) + # This is not great, but it's better to have some dependencies on these Python files. python_deps = [ '$(TOPSRCDIR)/python/mozbuild/mozbuild/action/l10n_merge.py', @@ -209,14 +208,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): for (merge, ref_file, l10n_file) in deps: rule = mk.create_rule([merge]).add_dependencies( [ref_file, l10n_file] + python_deps) - rule.add_commands( - [ - '$(PYTHON) -m mozbuild.action.l10n_merge ' - '--output {} --ref-file {} --l10n-file {}'.format( - merge, ref_file, l10n_file - ) - ] - ) + rule.add_commands(['$(PYTHON) -m mozbuild.action.l10n_merge --output {} --ref-file {} --l10n-file {}'.format(merge, ref_file, l10n_file)]) # Add a dummy rule for the l10n file since it might not exist. mk.create_rule([l10n_file]) @@ -228,8 +220,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend): 'install_%s' % base.replace('/', '_'))) as fh: install_manifest.write(fileobj=fh) - # For artifact builds only, write a single unified manifest - # for consumption by |mach watch|. + # For artifact builds only, write a single unified manifest for consumption by |mach watch|. if self.environment.is_artifact_build: unified_manifest = InstallManifest() for base, install_manifest in self._install_manifests.iteritems(): diff --git a/python/mozbuild/mozbuild/backend/mach_commands.py b/python/mozbuild/mozbuild/backend/mach_commands.py index dae4c548cb50..028fefcd0999 100644 --- a/python/mozbuild/mozbuild/backend/mach_commands.py +++ b/python/mozbuild/mozbuild/backend/mach_commands.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals import argparse import os +import sys import subprocess import which @@ -19,11 +20,10 @@ from mach.decorators import ( Command, ) - @CommandProvider class MachCommands(MachCommandBase): @Command('ide', category='devenv', - description='Generate a project and launch an IDE.') + description='Generate a project and launch an IDE.') @CommandArgument('ide', choices=['eclipse', 'visualstudio']) @CommandArgument('args', nargs=argparse.REMAINDER) def eclipse(self, ide, args): @@ -57,12 +57,10 @@ class MachCommands(MachCommandBase): if ide == 'eclipse': eclipse_workspace_dir = self.get_eclipse_workspace_path() - subprocess.check_call(['eclipse', '-data', eclipse_workspace_dir]) + process = subprocess.check_call(['eclipse', '-data', eclipse_workspace_dir]) elif ide == 'visualstudio': visual_studio_workspace_dir = self.get_visualstudio_workspace_path() - subprocess.check_call( - ['explorer.exe', visual_studio_workspace_dir] - ) + process = subprocess.check_call(['explorer.exe', visual_studio_workspace_dir]) def get_eclipse_workspace_path(self): from mozbuild.backend.cpp_eclipse import CppEclipseBackend diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 95770e83d866..d550cd876f53 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import logging import os @@ -51,6 +51,7 @@ from ..frontend.data import ( HostSources, InstallationTarget, JARManifest, + Library, Linkable, LocalInclude, LocalizedFiles, @@ -125,7 +126,7 @@ MOZBUILD_VARIABLES = [ b'TEST_DIRS', b'TOOL_DIRS', # XXX config/Makefile.in specifies this in a make invocation - # 'USE_EXTENSION_MANIFEST', + #'USE_EXTENSION_MANIFEST', b'XPCSHELL_TESTS', b'XPIDL_MODULE', ] @@ -224,9 +225,9 @@ class BackendMakeFile(object): self.fh.write('NONRECURSIVE_TARGETS += export\n') self.fh.write('NONRECURSIVE_TARGETS_export += xpidl\n') self.fh.write('NONRECURSIVE_TARGETS_export_xpidl_DIRECTORY = ' - '$(DEPTH)/xpcom/xpidl\n') + '$(DEPTH)/xpcom/xpidl\n') self.fh.write('NONRECURSIVE_TARGETS_export_xpidl_TARGETS += ' - 'export\n') + 'export\n') return self.fh.close() @@ -247,7 +248,6 @@ class RecursiveMakeTraversal(object): """ SubDirectoryCategories = ['dirs', 'tests'] SubDirectoriesTuple = namedtuple('SubDirectories', SubDirectoryCategories) - class SubDirectories(SubDirectoriesTuple): def __new__(self): return RecursiveMakeTraversal.SubDirectoriesTuple.__new__(self, [], []) @@ -312,7 +312,7 @@ class RecursiveMakeTraversal(object): if start_node != '': deps[start_node] = prev_nodes prev_nodes = (start_node,) - if start_node not in self._traversal: + if not start_node in self._traversal: return prev_nodes parallel_nodes = [] for node in parallel: @@ -338,7 +338,7 @@ class RecursiveMakeTraversal(object): current, parallel, sequential = self.call_filter(start, filter) if current is not None: yield start - if start not in self._traversal: + if not start in self._traversal: return for node in parallel: for n in self.traverse(node, filter): @@ -390,7 +390,7 @@ class RecursiveMakeBackend(CommonBackend): self._test_manifests = {} self.backend_input_files.add(mozpath.join(self.environment.topobjdir, - 'config', 'autoconf.mk')) + 'config', 'autoconf.mk')) self._install_manifests = defaultdict(InstallManifest) # The build system relies on some install manifests always existing @@ -424,7 +424,7 @@ class RecursiveMakeBackend(CommonBackend): if obj.objdir not in self._backend_files: self._backend_files[obj.objdir] = \ BackendMakeFile(obj.srcdir, obj.objdir, obj.config, - obj.topsrcdir, self.environment.topobjdir, self.dry_run) + obj.topsrcdir, self.environment.topobjdir, self.dry_run) return self._backend_files[obj.objdir] def consume_object(self, obj): @@ -603,30 +603,25 @@ class RecursiveMakeBackend(CommonBackend): backend_file.write('GARBAGE += %s\n' % stub_file) backend_file.write('EXTRA_MDDEPEND_FILES += %s\n' % dep_file) - backend_file.write(( - """{stub}: {script}{inputs}{backend}{force} + backend_file.write("""{stub}: {script}{inputs}{backend}{force} \t$(REPORT_BUILD) -\t$(call py_action,file_generate,{locale}{script} """ # wrap for E501 - """{method} {output} $(MDDEPDIR)/{dep_file} {stub}{inputs}{flags}) +\t$(call py_action,file_generate,{locale}{script} {method} {output} $(MDDEPDIR)/{dep_file} {stub}{inputs}{flags}) \t@$(TOUCH) $@ -""").format( - stub=stub_file, - output=first_output, - dep_file=dep_file, - inputs=' ' + ' '.join(inputs) if inputs else '', - flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '', - backend=' backend.mk' if obj.flags else '', - # Locale repacks repack multiple locales from a single configured objdir, - # so standard mtime dependencies won't work properly when the build is re-run - # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set - # in this situation, so simply force the generation to run in that case. - force=force, - locale='--locale=$(AB_CD) ' if obj.localized else '', - script=obj.script, - method=obj.method - ) - ) +""".format(stub=stub_file, + output=first_output, + dep_file=dep_file, + inputs=' ' + ' '.join(inputs) if inputs else '', + flags=' ' + ' '.join(shell_quote(f) for f in obj.flags) if obj.flags else '', + backend=' backend.mk' if obj.flags else '', + # Locale repacks repack multiple locales from a single configured objdir, + # so standard mtime dependencies won't work properly when the build is re-run + # with a different locale as input. IS_LANGUAGE_REPACK will reliably be set + # in this situation, so simply force the generation to run in that case. + force=force, + locale='--locale=$(AB_CD) ' if obj.localized else '', + script=obj.script, + method=obj.method)) elif isinstance(obj, JARManifest): self._no_skip['libs'].add(backend_file.relobjdir) @@ -789,7 +784,7 @@ class RecursiveMakeBackend(CommonBackend): main, all_deps = \ self._traversal.compute_dependencies(filter) for dir, deps in all_deps.items(): - if deps is not None or (dir in self._idl_dirs + if deps is not None or (dir in self._idl_dirs \ and tier == 'export'): rule = root_deps_mk.create_rule(['%s/%s' % (dir, tier)]) if deps: @@ -800,8 +795,8 @@ class RecursiveMakeBackend(CommonBackend): if main: rule.add_dependencies('%s/%s' % (d, tier) for d in main) - all_compile_deps = reduce(lambda x, y: x | y, - self._compile_graph.values()) if self._compile_graph else set() + all_compile_deps = reduce(lambda x,y: x|y, + self._compile_graph.values()) if self._compile_graph else set() # Include the following as dependencies of the top recursion target for # compilation: # - nodes that are not dependended upon by anything. Typically, this @@ -822,10 +817,8 @@ class RecursiveMakeBackend(CommonBackend): # Directories containing rust compilations don't generally depend # on other directories in the tree, so putting them first here will # start them earlier in the build. - rule.add_dependencies( - chain((r for r in roots if mozpath.dirname(r) in self._rust_dirs), - (r for r in roots if mozpath.dirname(r) not in self._rust_dirs)) - ) + rule.add_dependencies(chain((r for r in roots if mozpath.dirname(r) in self._rust_dirs), + (r for r in roots if mozpath.dirname(r) not in self._rust_dirs))) for target, deps in sorted(graph.items()): if deps: rule = root_deps_mk.create_rule([target]) @@ -908,8 +901,7 @@ class RecursiveMakeBackend(CommonBackend): all_sources)) if include_curdir_build_rules: - makefile.add_statement( - '\n' + makefile.add_statement('\n' '# Make sometimes gets confused between "foo" and "$(CURDIR)/foo".\n' '# Help it out by explicitly specifiying dependencies.') makefile.add_statement('all_absolute_unified_files := \\\n' @@ -957,7 +949,7 @@ class RecursiveMakeBackend(CommonBackend): stub = not os.path.exists(makefile_in) if not stub: self.log(logging.DEBUG, 'substitute_makefile', - {'path': makefile}, 'Substituting makefile: {path}') + {'path': makefile}, 'Substituting makefile: {path}') self._makefile_in_count += 1 # In the export and libs tiers, we don't skip directories @@ -969,7 +961,7 @@ class RecursiveMakeBackend(CommonBackend): self._no_skip[tier].add(bf.relobjdir) else: self.log(logging.DEBUG, 'stub_makefile', - {'path': makefile}, 'Creating stub Makefile: {path}') + {'path': makefile}, 'Creating stub Makefile: {path}') obj = self.Substitution() obj.output_path = makefile @@ -984,7 +976,7 @@ class RecursiveMakeBackend(CommonBackend): # XPI_PKGNAME or INSTALL_EXTENSION_ID can't be skipped and # must run during the 'tools' tier. for t in (b'XPI_PKGNAME', b'INSTALL_EXTENSION_ID', - b'tools'): + b'tools'): if t not in content: continue if t == b'tools' and not re.search('(?:^|\s)tools.*::', content, re.M): @@ -992,13 +984,13 @@ class RecursiveMakeBackend(CommonBackend): if objdir == self.environment.topobjdir: continue self._no_skip['tools'].add(mozpath.relpath(objdir, - self.environment.topobjdir)) + self.environment.topobjdir)) # Directories with a Makefile containing a check target # can't be skipped and must run during the 'check' tier. if re.search('(?:^|\s)check.*::', content, re.M): self._no_skip['check'].add(mozpath.relpath(objdir, - self.environment.topobjdir)) + self.environment.topobjdir)) # Detect any Makefile.ins that contain variables on the # moz.build-only list @@ -1075,10 +1067,10 @@ class RecursiveMakeBackend(CommonBackend): return (mozpath.relpath(d.translated, base) for d in dirs) if obj.dirs: - fh.write('DIRS := %s\n' % ' '.join(relativize(backend_file.objdir, obj.dirs))) - self._traversal.add( - backend_file.relobjdir, dirs=relativize(self.environment.topobjdir, obj.dirs) - ) + fh.write('DIRS := %s\n' % ' '.join( + relativize(backend_file.objdir, obj.dirs))) + self._traversal.add(backend_file.relobjdir, + dirs=relativize(self.environment.topobjdir, obj.dirs)) # The directory needs to be registered whether subdirectories have been # registered or not. @@ -1102,10 +1094,7 @@ class RecursiveMakeBackend(CommonBackend): if obj.target and not obj.is_custom(): backend_file.write('FINAL_TARGET = $(DEPTH)/%s\n' % (obj.target)) else: - backend_file.write( - 'FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),' - '$(DIST)/bin)$(DIST_SUBDIR:%=/%)\n' - ) + backend_file.write('FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)\n') if not obj.enabled: backend_file.write('NO_DIST_INSTALL := 1\n') @@ -1120,7 +1109,8 @@ class RecursiveMakeBackend(CommonBackend): self._install_manifests['dist_include'].add_optional_exists('%s.h' % stem) for module in manager.modules: - build_files.add_optional_exists(mozpath.join('.deps', '%s.pp' % module)) + build_files.add_optional_exists(mozpath.join('.deps', + '%s.pp' % module)) modules = manager.modules xpt_modules = sorted(modules.keys()) @@ -1160,9 +1150,9 @@ class RecursiveMakeBackend(CommonBackend): obj = self.Substitution() obj.output_path = mozpath.join(self.environment.topobjdir, 'config', - 'makefiles', 'xpidl', 'Makefile') + 'makefiles', 'xpidl', 'Makefile') obj.input_path = mozpath.join(self.environment.topsrcdir, 'config', - 'makefiles', 'xpidl', 'Makefile.in') + 'makefiles', 'xpidl', 'Makefile.in') obj.topsrcdir = self.environment.topsrcdir obj.topobjdir = self.environment.topobjdir obj.config = self.environment @@ -1246,7 +1236,7 @@ class RecursiveMakeBackend(CommonBackend): # Much of the logic in this function could be moved to CommonBackend. for source in obj.source_relpaths: self.backend_input_files.add(mozpath.join(obj.topsrcdir, - source)) + source)) # Don't allow files to be defined multiple times unless it is allowed. # We currently allow duplicates for non-test files or test files if @@ -1261,7 +1251,7 @@ class RecursiveMakeBackend(CommonBackend): for base, pattern, dest in obj.pattern_installs: try: self._install_manifests['_test_files'].add_pattern_link(base, - pattern, dest) + pattern, dest) except ValueError: if not obj.dupe_manifest: raise @@ -1274,7 +1264,7 @@ class RecursiveMakeBackend(CommonBackend): raise m = self._test_manifests.setdefault(obj.flavor, - (obj.install_prefix, set())) + (obj.install_prefix, set())) m[1].add(obj.manifest_obj_relpath) try: @@ -1305,14 +1295,12 @@ class RecursiveMakeBackend(CommonBackend): def _process_per_source_flag(self, per_source_flag, backend_file): for flag in per_source_flag.flags: - backend_file.write('%s_FLAGS += %s\n' % - (mozpath.basename(per_source_flag.file_name), flag)) + backend_file.write('%s_FLAGS += %s\n' % (mozpath.basename(per_source_flag.file_name), flag)) def _process_computed_flags(self, computed_flags, backend_file): for var, flags in computed_flags.get_flags(): - backend_file.write( - 'COMPUTED_%s += %s\n' % (var, - ' '.join(make_quote(shell_quote(f)) for f in flags))) + backend_file.write('COMPUTED_%s += %s\n' % (var, + ' '.join(make_quote(shell_quote(f)) for f in flags))) def _process_non_default_target(self, libdef, target_name, backend_file): backend_file.write("%s:: %s\n" % (libdef.output_category, target_name)) @@ -1369,13 +1357,14 @@ class RecursiveMakeBackend(CommonBackend): else: target_name = obj.KIND return '%s/%s' % (mozpath.relpath(obj.objdir, - self.environment.topobjdir), target_name) + self.environment.topobjdir), target_name) def _process_linked_libraries(self, obj, backend_file): def pretty_relpath(lib, name): return os.path.normpath(mozpath.join(mozpath.relpath(lib.objdir, obj.objdir), name)) + topobjdir = mozpath.normsep(obj.topobjdir) # This will create the node even if there aren't any linked libraries. build_target = self._build_target_for_obj(obj) self._compile_graph[build_target] @@ -1425,7 +1414,7 @@ class RecursiveMakeBackend(CommonBackend): # incorrect list file format to the host compiler as well as when # creating an archive with AR, which doesn't understand list files. if (objs == obj.objs and not isinstance(obj, (HostLibrary, StaticLibrary)) or - isinstance(obj, StaticLibrary) and obj.no_expand_lib): + isinstance(obj, StaticLibrary) and obj.no_expand_lib): backend_file.write_once('%s_OBJS := %s\n' % (obj.name, objs_ref)) if profile_gen_objs: @@ -1530,8 +1519,8 @@ class RecursiveMakeBackend(CommonBackend): if f.startswith('/') or isinstance(f, AbsolutePath): basepath, wild = os.path.split(f.full_path) if '*' in basepath: - raise Exception("Wildcards are only supported in the filename part" - " of srcdir-relative or absolute paths.") + raise Exception("Wildcards are only supported in the filename part of " + "srcdir-relative or absolute paths.") install_manifest.add_pattern_link(basepath, wild, path) else: @@ -1662,7 +1651,7 @@ class RecursiveMakeBackend(CommonBackend): def _write_manifests(self, dest, manifests): man_dir = mozpath.join(self.environment.topobjdir, '_build_manifests', - dest) + dest) for k, manifest in manifests.items(): with self._write_file(mozpath.join(man_dir, k)) as fh: @@ -1699,7 +1688,7 @@ class RecursiveMakeBackend(CommonBackend): pp.context.update(extra) if not pp.context.get('autoconfmk', ''): pp.context['autoconfmk'] = 'autoconf.mk' - pp.handleLine(b'# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n') + pp.handleLine(b'# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n'); pp.handleLine(b'DEPTH := @DEPTH@\n') pp.handleLine(b'topobjdir := @topobjdir@\n') pp.handleLine(b'topsrcdir := @top_srcdir@\n') @@ -1743,19 +1732,18 @@ class RecursiveMakeBackend(CommonBackend): rule.add_commands([ '$(RM) $@', '$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) ' - '$< -o $@)' + '$< -o $@)' ]) mk.add_statement('ALL_IPDLSRCS := %s %s' % (' '.join(sorted_nonstatic_ipdl_basenames), - ' '.join(sorted_static_ipdl_sources))) + ' '.join(sorted_static_ipdl_sources))) self._add_unified_build_rules(mk, unified_ipdl_cppsrcs_mapping, unified_files_makefile_variable='CPPSRCS') # Preprocessed ipdl files are generated in ipdl_dir. - mk.add_statement( - 'IPDLDIRS := %s %s' % (ipdl_dir, ' '.join(sorted(set(mozpath.dirname(p) - for p in sorted_static_ipdl_sources))))) + mk.add_statement('IPDLDIRS := %s %s' % (ipdl_dir, ' '.join(sorted(set(mozpath.dirname(p) + for p in sorted_static_ipdl_sources))))) with self._write_file(mozpath.join(ipdl_dir, 'ipdlsrcs.mk')) as ipdls: mk.dump(ipdls, removal_guard=False) @@ -1764,7 +1752,7 @@ class RecursiveMakeBackend(CommonBackend): webidls, expected_build_output_files, global_define_files): include_dir = mozpath.join(self.environment.topobjdir, 'dist', - 'include') + 'include') for f in expected_build_output_files: if f.startswith(include_dir): self._install_manifests['dist_include'].add_optional_exists( @@ -1795,12 +1783,12 @@ class RecursiveMakeBackend(CommonBackend): # which would modify content in the source directory. '$(RM) $@', '$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) ' - '$< -o $@)' + '$< -o $@)' ]) self._add_unified_build_rules(mk, - unified_source_mapping, - unified_files_makefile_variable='unified_binding_cpp_files') + unified_source_mapping, + unified_files_makefile_variable='unified_binding_cpp_files') webidls_mk = mozpath.join(bindings_dir, 'webidlsrcs.mk') with self._write_file(webidls_mk) as fh: diff --git a/python/mozbuild/mozbuild/backend/test_manifest.py b/python/mozbuild/mozbuild/backend/test_manifest.py index 8cfba0ef5774..218d81a83575 100644 --- a/python/mozbuild/mozbuild/backend/test_manifest.py +++ b/python/mozbuild/mozbuild/backend/test_manifest.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import cPickle as pickle from collections import defaultdict diff --git a/python/mozbuild/mozbuild/backend/tup.py b/python/mozbuild/mozbuild/backend/tup.py index 4bdab6df065f..046586ddc325 100644 --- a/python/mozbuild/mozbuild/backend/tup.py +++ b/python/mozbuild/mozbuild/backend/tup.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import os import gzip @@ -12,9 +12,12 @@ import sys import shutil import mozpack.path as mozpath +from mozbuild import shellutil from mozbuild.analyze.graph import Graph from mozbuild.analyze.hg import Report from mozbuild.base import MozbuildObject +from mozbuild.backend.base import PartialBackend, HybridBackend +from mozbuild.backend.recursivemake import RecursiveMakeBackend from mozbuild.mozconfig import MozconfigLoader from mozbuild.shellutil import quote as shell_quote from mozbuild.util import OrderedDefaultDict @@ -56,6 +59,7 @@ from ..frontend.data import ( ) from ..util import ( FileAvoidWrite, + expand_variables, ) from ..frontend.context import ( AbsolutePath, @@ -143,8 +147,7 @@ class BackendTupfile(object): else: caret_text = flags - self.write((': %(inputs)s%(extra_inputs)s |> %(display)s%(cmd)s |> ' - '%(outputs)s%(output_group)s\n') % { + self.write(': %(inputs)s%(extra_inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(output_group)s\n' % { 'inputs': ' '.join(inputs), 'extra_inputs': ' | ' + ' '.join(extra_inputs) if extra_inputs else '', 'display': '^%s^ ' % caret_text if caret_text else '', @@ -269,8 +272,7 @@ class TupBackend(CommonBackend): self._rust_cmds = set() self._built_in_addons = set() - self._built_in_addons_file = \ - 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json' + self._built_in_addons_file = 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json' def _output_group(self, label): if label: @@ -330,9 +332,9 @@ class TupBackend(CommonBackend): tiers.set_tiers(('tup',)) tiers.begin_tier('tup') status = config.run_process(args=args, - line_handler=output.on_line, - ensure_exit_code=False, - append_env=self._get_mozconfig_env(config)) + line_handler=output.on_line, + ensure_exit_code=False, + append_env=self._get_mozconfig_env(config)) tiers.finish_tier('tup') if not status and self.environment.substs.get('MOZ_AUTOMATION'): config.log_manager.enable_unstructured() @@ -473,6 +475,7 @@ class TupBackend(CommonBackend): # accurate once we start building libraries in their final locations. inputs = objs + static_libs + shared_libs + [self._shlibs] + rust_linked = [l for l in prog.linked_libraries if isinstance(l, RustLibrary)] @@ -510,10 +513,12 @@ class TupBackend(CommonBackend): display='LINK %o' ) + def _gen_host_programs(self, backend_file): for p in backend_file.host_programs: self._gen_host_program(backend_file, p) + def _gen_host_program(self, backend_file, prog): _, _, _, _, extra_libs, _ = self._expand_libs(prog) objs = prog.objs @@ -554,6 +559,7 @@ class TupBackend(CommonBackend): display='LINK %o' ) + def _gen_static_library(self, backend_file): ar = [ backend_file.environment.substs['AR'], @@ -578,6 +584,7 @@ class TupBackend(CommonBackend): display='AR %o' ) + def consume_object(self, obj): """Write out build files necessary to build with tup.""" @@ -670,13 +677,11 @@ class TupBackend(CommonBackend): for objdir, backend_file in sorted(self._backend_files.items()): backend_file.gen_sources_rules([self._installed_files]) - for var, gen_method in ( - (backend_file.shared_lib, self._gen_shared_library), - (backend_file.static_lib and backend_file.static_lib.no_expand_lib, - self._gen_static_library), - (backend_file.programs, self._gen_programs), - (backend_file.host_programs, self._gen_host_programs) - ): + for var, gen_method in ((backend_file.shared_lib, self._gen_shared_library), + (backend_file.static_lib and backend_file.static_lib.no_expand_lib, + self._gen_static_library), + (backend_file.programs, self._gen_programs), + (backend_file.host_programs, self._gen_host_programs)): if var: backend_file.export_shell() backend_file.export_icecc() @@ -689,9 +694,8 @@ class TupBackend(CommonBackend): pass with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh: - acdefines_flags = ' '.join( - ['-D%s=%s' % (name, shell_quote(value)) - for (name, value) in sorted(self.environment.acdefines.iteritems())]) + acdefines_flags = ' '.join(['-D%s=%s' % (name, shell_quote(value)) + for (name, value) in sorted(self.environment.acdefines.iteritems())]) # TODO: AB_CD only exists in Makefiles at the moment. acdefines_flags += ' -DAB_CD=en-US' @@ -725,13 +729,12 @@ class TupBackend(CommonBackend): # Ask the user to figure out where to run 'tup init' before # continuing. raise Exception("Please run `tup init --no-sync` in a common " - "ancestor directory of your objdir and srcdir, possibly " - "%s. To reduce file scanning overhead, this directory " - "should contain the fewest files possible that are not " - "necessary for this build." % tup_base_dir) + "ancestor directory of your objdir and srcdir, possibly " + "%s. To reduce file scanning overhead, this directory " + "should contain the fewest files possible that are not " + "necessary for this build." % tup_base_dir) tup = self.environment.substs.get('TUP', 'tup') - self._cmd.run_process(cwd=tup_base_dir, log_name='tup', - args=[tup, 'init', '--no-sync']) + self._cmd.run_process(cwd=tup_base_dir, log_name='tup', args=[tup, 'init', '--no-sync']) def _get_cargo_flags(self, obj): @@ -810,11 +813,13 @@ class TupBackend(CommonBackend): # Enable link-time optimization for release builds. cargo_library_flags = [] - if not obj.config.substs.get('DEVELOPER_OPTIONS') and not obj.config.substs.get( - 'MOZ_DEBUG_RUST' - ): + if (not obj.config.substs.get('DEVELOPER_OPTIONS') and + not obj.config.substs.get('MOZ_DEBUG_RUST')): cargo_library_flags += ['-C', 'lto'] + rust_build_home = mozpath.join(self.environment.topobjdir, + 'toolkit/library/rust') + def display_name(invocation): output_str = '' if invocation['outputs']: @@ -868,9 +873,7 @@ class TupBackend(CommonBackend): invocation['full-deps'] = set() - if os.path.basename(invocation['program']) in [ - 'build-script-build', 'build-script-main' - ]: + if os.path.basename(invocation['program']) in ['build-script-build', 'build-script-main']: out_dir = invocation['env']['OUT_DIR'] for output in cargo_extra_outputs.get(shortname, []): outputs.append(os.path.join(out_dir, output)) @@ -981,15 +984,16 @@ class TupBackend(CommonBackend): obj.name), output_group) + for val in enumerate(invocations): _process(*val) + def _gen_rust_rules(self, obj, backend_file): cargo_flags = self._get_cargo_flags(obj) cargo_env = self._get_cargo_env(obj, backend_file) output_lines = [] - def accumulate_output(line): output_lines.append(line) @@ -1010,6 +1014,7 @@ class TupBackend(CommonBackend): self._gen_cargo_rules(obj, cargo_plan, cargo_env, output_group) self.backend_input_files |= set(cargo_plan['inputs']) + def _process_generated_file(self, backend_file, obj): if obj.script and obj.method: backend_file.export_shell() @@ -1020,8 +1025,8 @@ class TupBackend(CommonBackend): obj.script, obj.method, obj.outputs[0], - '%s.pp' % obj.outputs[0], # deps file required - 'unused', # deps target is required + '%s.pp' % obj.outputs[0], # deps file required + 'unused', # deps target is required ]) full_inputs = [f.full_path for f in obj.inputs] cmd.extend(full_inputs) @@ -1120,9 +1125,8 @@ class TupBackend(CommonBackend): if f.startswith('/') or isinstance(f, AbsolutePath): basepath, wild = os.path.split(f.full_path) if '*' in basepath: - raise Exception( - "Wildcards are only supported in the filename part of " - "srcdir-relative or absolute paths.") + raise Exception("Wildcards are only supported in the filename part of " + "srcdir-relative or absolute paths.") # TODO: This is only needed for Windows, so we can # skip this for now. @@ -1151,6 +1155,7 @@ class TupBackend(CommonBackend): finder = FileFinder(prefix) for p, _ in finder.find(f.full_path[len(prefix):]): + install_dir = prefix[len(obj.srcdir) + 1:] output = p if f.target_basename and '*' not in f.target_basename: output = mozpath.join(f.target_basename, output) @@ -1158,12 +1163,10 @@ class TupBackend(CommonBackend): output=mozpath.join(output_dir, output), output_group=output_group) else: - backend_file.symlink_rule( - f.full_path, output=f.target_basename, output_group=output_group) + backend_file.symlink_rule(f.full_path, output=f.target_basename, output_group=output_group) else: if (self.environment.is_artifact_build and - any(mozpath.match(f.target_basename, p) - for p in self._compile_env_gen_files)): + any(mozpath.match(f.target_basename, p) for p in self._compile_env_gen_files)): # If we have an artifact build we never would have generated this file, # so do not attempt to install it. continue @@ -1172,19 +1175,18 @@ class TupBackend(CommonBackend): f.target_basename) gen_backend_file = self._get_backend_file(f.context.relobjdir) if gen_backend_file.requires_delay([f]): - gen_backend_file.delayed_installed_files.append( - (f.full_path, output, output_group)) + gen_backend_file.delayed_installed_files.append((f.full_path, output, output_group)) else: gen_backend_file.symlink_rule(f.full_path, output=output, output_group=output_group) + def _process_final_target_pp_files(self, obj, backend_file): for i, (path, files) in enumerate(obj.files.walk()): self._add_features(obj.install_target, path) for f in files: self._preprocess(backend_file, f.full_path, - destdir=mozpath.join(self.environment.topobjdir, - obj.install_target, path), + destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path), target=f.target_basename) def _process_computed_flags(self, obj, backend_file): @@ -1313,8 +1315,7 @@ class TupBackend(CommonBackend): cmd.extend(['-I%s' % d for d in ipdldirs]) cmd.extend(sorted_ipdl_sources) - outputs = ['IPCMessageTypeName.cpp', mozpath.join( - outheaderdir, 'IPCMessageStart.h'), 'ipdl_lextab.py', 'ipdl_yacctab.py'] + outputs = ['IPCMessageTypeName.cpp', mozpath.join(outheaderdir, 'IPCMessageStart.h'), 'ipdl_lextab.py', 'ipdl_yacctab.py'] for filename in sorted_ipdl_sources: filepath, ext = os.path.splitext(filename) @@ -1378,5 +1379,4 @@ class TupBackend(CommonBackend): backend_file.sources['.cpp'].extend(sorted(global_define_files)) test_backend_file = self._get_backend_file('dom/bindings/test') - test_backend_file.sources['.cpp'].extend( - sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems())) + test_backend_file.sources['.cpp'].extend(sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems())) diff --git a/python/mozbuild/mozbuild/backend/visualstudio.py b/python/mozbuild/mozbuild/backend/visualstudio.py index 24cd3b6e644b..410b47351a2d 100644 --- a/python/mozbuild/mozbuild/backend/visualstudio.py +++ b/python/mozbuild/mozbuild/backend/visualstudio.py @@ -5,11 +5,12 @@ # This file contains a build backend for generating Visual Studio project # files. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import errno import os import re +import types import uuid from xml.dom import getDOMImplementation @@ -33,25 +34,21 @@ from mozbuild.base import ExecutionSummary MSBUILD_NAMESPACE = 'http://schemas.microsoft.com/developer/msbuild/2003' - def get_id(name): return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper() - def visual_studio_product_to_solution_version(version): if version == '2017': return '12.00', '15' else: raise Exception('Unknown version seen: %s' % version) - def visual_studio_product_to_platform_toolset_version(version): if version == '2017': return 'v141' else: raise Exception('Unknown version seen: %s' % version) - class VisualStudioBackend(CommonBackend): """Generate Visual Studio project files. @@ -103,7 +100,7 @@ class VisualStudioBackend(CommonBackend): elif isinstance(obj, UnifiedSources): # XXX we should be letting CommonBackend.consume_object call this # for us instead. - self._process_unified_sources(obj) + self._process_unified_sources(obj); elif isinstance(obj, Library): self._libs_to_paths[obj.basename] = reldir @@ -136,9 +133,9 @@ class VisualStudioBackend(CommonBackend): out_proj_dir = os.path.join(self._out_dir, self._projsubdir) projects = self._write_projects_for_sources(self._libs_to_paths, - "library", out_proj_dir) + "library", out_proj_dir) projects.update(self._write_projects_for_sources(self._progs_to_paths, - "binary", out_proj_dir)) + "binary", out_proj_dir)) # Generate projects that can be used to build common targets. for target in ('export', 'binaries', 'tools', 'full'): @@ -147,16 +144,15 @@ class VisualStudioBackend(CommonBackend): if target != 'full': command += ' %s' % target - project_id = self._write_vs_project( - out_proj_dir, basename, target, build_command=command, + project_id = self._write_vs_project(out_proj_dir, basename, target, + build_command=command, clean_command='$(SolutionDir)\\mach.bat build clean') projects[basename] = (project_id, basename, target) # A project that can be used to regenerate the visual studio projects. basename = 'target_vs' - project_id = self._write_vs_project( - out_proj_dir, basename, 'visual-studio', + project_id = self._write_vs_project(out_proj_dir, basename, 'visual-studio', build_command='$(SolutionDir)\\mach.bat build-backend -b VisualStudio') projects[basename] = (project_id, basename, 'visual-studio') @@ -194,7 +190,7 @@ class VisualStudioBackend(CommonBackend): headers = [t[0] for t in finder.find('*.h')] headers = [os.path.normpath(os.path.join('$(TopSrcDir)', - path, f)) for f in headers] + path, f)) for f in headers] includes = [ os.path.join('$(TopSrcDir)', path), @@ -205,7 +201,7 @@ class VisualStudioBackend(CommonBackend): includes.append('$(TopObjDir)\\dist\\include') for v in ('NSPR_CFLAGS', 'NSS_CFLAGS', 'MOZ_JPEG_CFLAGS', - 'MOZ_PNG_CFLAGS', 'MOZ_ZLIB_CFLAGS', 'MOZ_PIXMAN_CFLAGS'): + 'MOZ_PNG_CFLAGS', 'MOZ_ZLIB_CFLAGS', 'MOZ_PIXMAN_CFLAGS'): if not config: break @@ -227,7 +223,7 @@ class VisualStudioBackend(CommonBackend): else: defines.append('%s=%s' % (k, v)) - debugger = None + debugger=None if prefix == 'binary': if item.startswith(self.environment.substs['MOZ_APP_NAME']): app_args = '-no-remote -profile $(TopObjDir)\\tmp\\profile-default' @@ -239,8 +235,7 @@ class VisualStudioBackend(CommonBackend): basename = '%s_%s' % (prefix, item) - project_id = self._write_vs_project( - out_dir, basename, item, + project_id = self._write_vs_project(out_dir, basename, item, includes=includes, forced_includes=['$(TopObjDir)\\dist\\include\\mozilla-config.h'], defines=defines, @@ -421,7 +416,7 @@ class VisualStudioBackend(CommonBackend): fh.write(b'$env:%s = "%s"\r\n' % (k, v)) relpath = os.path.relpath(self.environment.topsrcdir, - self.environment.topobjdir).replace('\\', '/') + self.environment.topobjdir).replace('\\', '/') fh.write(b'$bashargs = "%s/mach", "--log-no-times"\r\n' % relpath) fh.write(b'$bashargs = $bashargs + $args\r\n') @@ -430,9 +425,9 @@ class VisualStudioBackend(CommonBackend): fh.write(b'$procargs = "-c", $expanded\r\n') fh.write(b'Start-Process -WorkingDirectory $env:TOPOBJDIR ' - b'-FilePath $env:MOZILLABUILD\\msys\\bin\\bash ' - b'-ArgumentList $procargs ' - b'-Wait -NoNewWindow\r\n') + b'-FilePath $env:MOZILLABUILD\\msys\\bin\\bash ' + b'-ArgumentList $procargs ' + b'-Wait -NoNewWindow\r\n') def _write_mach_batch(self, fh): """Write out a batch script that builds the tree. @@ -450,34 +445,34 @@ class VisualStudioBackend(CommonBackend): # relative paths, since munging c:\ to /c/ is slightly more # complicated. relpath = os.path.relpath(self.environment.topsrcdir, - self.environment.topobjdir).replace('\\', '/') + self.environment.topobjdir).replace('\\', '/') # We go through mach because it has the logic for choosing the most # appropriate build tool. fh.write(b'"%%MOZILLABUILD%%\\msys\\bin\\bash" ' - b'-c "%s/mach --log-no-times %%1 %%2 %%3 %%4 %%5 %%6 %%7"' % relpath) + b'-c "%s/mach --log-no-times %%1 %%2 %%3 %%4 %%5 %%6 %%7"' % relpath) def _write_vs_project(self, out_dir, basename, name, **kwargs): root = '%s.vcxproj' % basename project_id = get_id(basename.encode('utf-8')) with self._write_file(os.path.join(out_dir, root), mode='rb') as fh: - project_id, name = VisualStudioBackend.write_vs_project( - fh, self._version, project_id, name, **kwargs) + project_id, name = VisualStudioBackend.write_vs_project(fh, + self._version, project_id, name, **kwargs) with self._write_file(os.path.join(out_dir, '%s.user' % root), mode='rb') as fh: fh.write('\r\n') fh.write('\r\n' % - MSBUILD_NAMESPACE) + MSBUILD_NAMESPACE) fh.write('\r\n') return project_id @staticmethod def write_vs_project(fh, version, project_id, name, includes=[], - forced_includes=[], defines=[], - build_command=None, clean_command=None, - debugger=None, headers=[], sources=[]): + forced_includes=[], defines=[], + build_command=None, clean_command=None, + debugger=None, headers=[], sources=[]): impl = getDOMImplementation() doc = impl.createDocument(MSBUILD_NAMESPACE, 'Project', None) @@ -515,8 +510,7 @@ class VisualStudioBackend(CommonBackend): rn.appendChild(doc.createTextNode('mozilla')) pts = pg.appendChild(doc.createElement('PlatformToolset')) - pts.appendChild(doc.createTextNode( - visual_studio_product_to_platform_toolset_version(version))) + pts.appendChild(doc.createTextNode(visual_studio_product_to_platform_toolset_version(version))) i = project.appendChild(doc.createElement('Import')) i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.Default.props') diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py index abc61536abae..b0d7ac8432cd 100644 --- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -37,6 +37,7 @@ from .mozconfig import ( ) from .pythonutil import find_python3_executable from .util import ( + ReadOnlyNamespace, memoize, memoized_property, ) @@ -52,14 +53,12 @@ def ancestors(path): break path = newpath - def samepath(path1, path2): if hasattr(os.path, 'samefile'): return os.path.samefile(path1, path2) return os.path.normcase(os.path.realpath(path1)) == \ os.path.normcase(os.path.realpath(path2)) - class BadEnvironmentException(Exception): """Base class for errors raised when the build environment is not sane.""" @@ -70,7 +69,6 @@ class BuildEnvironmentNotFoundException(BadEnvironmentException): class ObjdirMismatchException(BadEnvironmentException): """Raised when the current dir is an objdir and doesn't match the mozconfig.""" - def __init__(self, objdir1, objdir2): self.objdir1 = objdir1 self.objdir2 = objdir2 @@ -87,7 +85,6 @@ class MozbuildObject(ProcessExecutionMixin): running processes, etc. This classes provides that functionality. Other modules can inherit from this class to obtain this functionality easily. """ - def __init__(self, topsrcdir, settings, log_manager, topobjdir=None, mozconfig=MozconfigLoader.AUTODETECT): """Create a new Mozbuild object instance. @@ -178,8 +175,7 @@ class MozbuildObject(ProcessExecutionMixin): topobjdir = mozpath.normsep(os.path.normpath(topobjdir)) if topsrcdir == topobjdir: - raise BadEnvironmentException( - 'The object directory appears ' + raise BadEnvironmentException('The object directory appears ' 'to be the same as your source directory (%s). This build ' 'configuration is not supported.' % topsrcdir) @@ -195,7 +191,7 @@ class MozbuildObject(ProcessExecutionMixin): if '@CONFIG_GUESS@' in topobjdir: topobjdir = topobjdir.replace('@CONFIG_GUESS@', - self.resolve_config_guess()) + self.resolve_config_guess()) if not os.path.isabs(topobjdir): topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir)) @@ -256,13 +252,10 @@ class MozbuildObject(ProcessExecutionMixin): @property def virtualenv_manager(self): if self._virtualenv_manager is None: - self._virtualenv_manager = VirtualenvManager( - self.topsrcdir, - self.topobjdir, - os.path.join(self.topobjdir, '_virtualenvs', 'init'), - sys.stdout, - os.path.join(self.topsrcdir, 'build', 'virtualenv_packages.txt') - ) + self._virtualenv_manager = VirtualenvManager(self.topsrcdir, + self.topobjdir, os.path.join(self.topobjdir, '_virtualenvs', 'init'), + sys.stdout, os.path.join(self.topsrcdir, 'build', + 'virtualenv_packages.txt')) return self._virtualenv_manager @@ -497,6 +490,7 @@ class MozbuildObject(ProcessExecutionMixin): return BuildReader(config, finder=finder) + @memoized_property def python3(self): """Obtain info about a Python 3 executable. @@ -548,10 +542,10 @@ class MozbuildObject(ProcessExecutionMixin): if substs['OS_ARCH'] == 'Darwin': if substs['MOZ_BUILD_APP'] == 'xulrunner': - stem = os.path.join(stem, 'XUL.framework') + stem = os.path.join(stem, 'XUL.framework'); else: stem = os.path.join(stem, substs['MOZ_MACBUNDLE_NAME'], 'Contents', - 'MacOS') + 'MacOS') elif where == 'default': stem = os.path.join(stem, 'bin') @@ -584,14 +578,13 @@ class MozbuildObject(ProcessExecutionMixin): notifier = which.which('terminal-notifier') except which.WhichError: raise Exception('Install terminal-notifier to get ' - 'a notification when the build finishes.') + 'a notification when the build finishes.') self.run_process([notifier, '-title', - 'Mozilla Build System', '-group', 'mozbuild', - '-message', msg], ensure_exit_code=False) + 'Mozilla Build System', '-group', 'mozbuild', + '-message', msg], ensure_exit_code=False) elif sys.platform.startswith('win'): from ctypes import Structure, windll, POINTER, sizeof from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT - class FLASHWINDOW(Structure): _fields_ = [("cbSize", UINT), ("hwnd", HANDLE), @@ -611,21 +604,21 @@ class MozbuildObject(ProcessExecutionMixin): return params = FLASHWINDOW(sizeof(FLASHWINDOW), - console, - FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) + console, + FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) FlashWindowEx(params) else: try: notifier = which.which('notify-send') except which.WhichError: raise Exception('Install notify-send (usually part of ' - 'the libnotify package) to get a notification when ' - 'the build finishes.') + 'the libnotify package) to get a notification when ' + 'the build finishes.') self.run_process([notifier, '--app-name=Mozilla Build System', - 'Mozilla Build System', msg], ensure_exit_code=False) + 'Mozilla Build System', msg], ensure_exit_code=False) except Exception as e: - self.log(logging.WARNING, 'notifier-failed', - {'error': e.message}, 'Notification center failed: {error}') + self.log(logging.WARNING, 'notifier-failed', {'error': + e.message}, 'Notification center failed: {error}') def _ensure_objdir_exists(self): if os.path.isdir(self.statedir): @@ -653,10 +646,10 @@ class MozbuildObject(ProcessExecutionMixin): return PathArgument(arg, self.topsrcdir, self.topobjdir) def _run_make(self, directory=None, filename=None, target=None, log=True, - srcdir=False, allow_parallel=True, line_handler=None, - append_env=None, explicit_env=None, ignore_errors=False, - ensure_exit_code=0, silent=True, print_directory=True, - pass_thru=False, num_jobs=0, keep_going=False): + srcdir=False, allow_parallel=True, line_handler=None, + append_env=None, explicit_env=None, ignore_errors=False, + ensure_exit_code=0, silent=True, print_directory=True, + pass_thru=False, num_jobs=0, keep_going=False): """Invoke make. directory -- Relative directory to look for Makefile in. @@ -798,11 +791,11 @@ class MozbuildObject(ProcessExecutionMixin): if xcode_lisense_error: raise Exception('Xcode requires accepting to the license agreement.\n' - 'Please run Xcode and accept the license agreement.') + 'Please run Xcode and accept the license agreement.') if self._is_windows(): raise Exception('Could not find a suitable make implementation.\n' - 'Please use MozillaBuild 1.9 or newer') + 'Please use MozillaBuild 1.9 or newer') else: raise Exception('Could not find a suitable make implementation.') @@ -827,12 +820,13 @@ class MozbuildObject(ProcessExecutionMixin): """ return cls(self.topsrcdir, self.settings, self.log_manager, - topobjdir=self.topobjdir) + topobjdir=self.topobjdir) def _activate_virtualenv(self): self.virtualenv_manager.ensure() self.virtualenv_manager.activate() + def _set_log_level(self, verbose): self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG) @@ -841,8 +835,7 @@ class MozbuildObject(ProcessExecutionMixin): pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv') if not os.path.exists(pipenv): for package in ['certifi', 'pipenv', 'six', 'virtualenv', 'virtualenv-clone']: - path = os.path.normpath(os.path.join( - self.topsrcdir, 'third_party/python', package)) + path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party/python', package)) self.virtualenv_manager.install_pip_package(path, vendored=True) return pipenv @@ -868,10 +861,9 @@ class MachCommandBase(MozbuildObject): detect_virtualenv_mozinfo = True if hasattr(context, 'detect_virtualenv_mozinfo'): detect_virtualenv_mozinfo = getattr(context, - 'detect_virtualenv_mozinfo') + 'detect_virtualenv_mozinfo') try: - dummy = MozbuildObject.from_environment( - cwd=context.cwd, + dummy = MozbuildObject.from_environment(cwd=context.cwd, detect_virtualenv_mozinfo=detect_virtualenv_mozinfo) topsrcdir = dummy.topsrcdir topobjdir = dummy._topobjdir @@ -889,12 +881,12 @@ class MachCommandBase(MozbuildObject): pass except ObjdirMismatchException as e: print('Ambiguous object directory detected. We detected that ' - 'both %s and %s could be object directories. This is ' - 'typically caused by having a mozconfig pointing to a ' - 'different object directory from the current working ' - 'directory. To solve this problem, ensure you do not have a ' - 'default mozconfig in searched paths.' % (e.objdir1, - e.objdir2)) + 'both %s and %s could be object directories. This is ' + 'typically caused by having a mozconfig pointing to a ' + 'different object directory from the current working ' + 'directory. To solve this problem, ensure you do not have a ' + 'default mozconfig in searched paths.' % (e.objdir1, + e.objdir2)) sys.exit(1) except MozconfigLoadException as e: @@ -911,7 +903,7 @@ class MachCommandBase(MozbuildObject): sys.exit(1) MozbuildObject.__init__(self, topsrcdir, context.settings, - context.log_manager, topobjdir=topobjdir) + context.log_manager, topobjdir=topobjdir) self._mach_context = context diff --git a/python/mozbuild/mozbuild/chunkify.py b/python/mozbuild/mozbuild/chunkify.py index c8b4051a2a81..da4b0791dae5 100644 --- a/python/mozbuild/mozbuild/chunkify.py +++ b/python/mozbuild/mozbuild/chunkify.py @@ -7,8 +7,6 @@ # of version 1.2. Its license (MPL2) is contained in repo root LICENSE file. # Please make modifications there where possible. -from __future__ import absolute_import, print_function - from itertools import islice @@ -56,3 +54,4 @@ def chunkify(things, this_chunk, chunks): return things[start:end] except TypeError: return islice(things, start, end) + diff --git a/python/mozbuild/mozbuild/codecoverage/chrome_map.py b/python/mozbuild/mozbuild/codecoverage/chrome_map.py index 5d5d97f9dfe6..eb5fbab9fbe6 100644 --- a/python/mozbuild/mozbuild/codecoverage/chrome_map.py +++ b/python/mozbuild/mozbuild/codecoverage/chrome_map.py @@ -2,11 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - +from collections import defaultdict import json import os import re +import urlparse from mach.config import ConfigSettings from mach.logging import LoggingManager @@ -22,12 +22,10 @@ from mozpack.files import PreprocessedFile from mozpack.manifests import InstallManifest import mozpack.path as mozpath -from .manifest_handler import ChromeManifestHandler +from manifest_handler import ChromeManifestHandler _line_comment_re = re.compile('^//@line (\d+) "(.+)"$') - - def generate_pp_info(path, topsrcdir): with open(path) as fh: # (start, end) -> (included_source, start) @@ -59,8 +57,6 @@ def generate_pp_info(path, topsrcdir): # This build backend is assuming the build to have happened already, as it is parsing # built preprocessed files to generate data to map them to the original sources. - - class ChromeMapBackend(CommonBackend): def _init(self): CommonBackend._init(self) @@ -120,11 +116,9 @@ class ChromeMapBackend(CommonBackend): # A map from url prefixes to objdir directories: # { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... } # A map of overrides. - # A map from objdir paths to sourcedir paths, and an object storing mapping - # information for preprocessed files: + # A map from objdir paths to sourcedir paths, and an object storing mapping information for preprocessed files: # { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js": - # [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ], - # ... } + # [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", {} ], ... } # An object containing build configuration information. outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json') with self._write_file(outputfile) as fh: diff --git a/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py b/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py index 982886999904..ef83388f2141 100644 --- a/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py +++ b/python/mozbuild/mozbuild/codecoverage/lcov_rewriter.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from argparse import ArgumentParser import json import os @@ -17,8 +15,7 @@ from six import viewitems from mozpack.chrome.manifest import parse_manifest import mozpack.path as mozpath -from .manifest_handler import ChromeManifestHandler - +from manifest_handler import ChromeManifestHandler class LcovRecord(object): __slots__ = ("test_name", @@ -33,7 +30,6 @@ class LcovRecord(object): "lines", "line_count", "covered_line_count") - def __init__(self): self.functions = {} self.function_exec_counts = {} @@ -68,16 +64,14 @@ class LcovRecord(object): # Re-calculate summaries after generating or splitting a record. self.function_count = len(self.functions.keys()) # Function records may have moved between files, so filter here. - self.function_exec_counts = { - fn_name: count for fn_name, count in viewitems(self.function_exec_counts) - if fn_name in self.functions.values()} + self.function_exec_counts = {fn_name: count for fn_name, count in viewitems(self.function_exec_counts) + if fn_name in self.functions.values()} self.covered_function_count = len([c for c in self.function_exec_counts.values() if c]) self.line_count = len(self.lines) self.covered_line_count = len([c for c, _ in self.lines.values() if c]) self.branch_count = len(self.branches) self.covered_branch_count = len([c for c in self.branches.values() if c]) - class RecordRewriter(object): # Helper class for rewriting/spliting individual lcov records according # to what the preprocessor did. @@ -170,8 +164,7 @@ class RecordRewriter(object): def rewrite_record(self, record, pp_info): # Rewrite the lines in the given record according to preprocessor info # and split to additional records when pp_info has included file info. - self._current_pp_info = dict( - [(tuple([int(l) for l in k.split(',')]), v) for k, v in pp_info.items()]) + self._current_pp_info = dict([(tuple([int(l) for l in k.split(',')]), v) for k, v in pp_info.items()]) self._ranges = sorted(self._current_pp_info.keys()) self._additions = {} self._rewrite_lines(record) @@ -185,7 +178,6 @@ class RecordRewriter(object): r.resummarize() return generated_records - class LcovFile(object): # Simple parser/pretty-printer for lcov format. # lcov parsing based on http://ltp.sourceforge.net/coverage/lcov/geninfo.1.php @@ -412,7 +404,6 @@ class LcovFile(object): class UrlFinderError(Exception): pass - class UrlFinder(object): # Given a "chrome://" or "resource://" url, uses data from the UrlMapBackend # and install manifests to find a path to the source file and the corresponding @@ -578,10 +569,10 @@ class UrlFinder(object): if app_name in url: if omnijar_name in url: - # e.g. file:///home/worker/workspace/build/application/firefox/omni.ja!/components/MainProcessSingleton.js # noqa + # e.g. file:///home/worker/workspace/build/application/firefox/omni.ja!/components/MainProcessSingleton.js parts = url_obj.path.split(omnijar_name + '!', 1) elif '.xpi!' in url: - # e.g. file:///home/worker/workspace/build/application/firefox/browser/features/e10srollout@mozilla.org.xpi!/bootstrap.js # noqa + # e.g. file:///home/worker/workspace/build/application/firefox/browser/features/e10srollout@mozilla.org.xpi!/bootstrap.js parts = url_obj.path.split('.xpi!', 1) else: # We don't know how to handle this jar: path, so return it to the @@ -589,10 +580,7 @@ class UrlFinder(object): return url_obj.path, None dir_parts = parts[0].rsplit(app_name + '/', 1) - url = mozpath.normpath( - mozpath.join(self.topobjdir, 'dist', - 'bin', dir_parts[1].lstrip('/'), parts[1].lstrip('/')) - ) + url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'bin', dir_parts[1].lstrip('/'), parts[1].lstrip('/'))) elif '.xpi!' in url: # This matching mechanism is quite brittle and based on examples seen in the wild. # There's no rule to match the XPI name to the path in dist/xpi-stage. @@ -602,8 +590,7 @@ class UrlFinder(object): addon_name = addon_name[:-len('-test@mozilla.org')] elif addon_name.endswith('@mozilla.org'): addon_name = addon_name[:-len('@mozilla.org')] - url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', - 'xpi-stage', addon_name, parts[1].lstrip('/'))) + url = mozpath.normpath(mozpath.join(self.topobjdir, 'dist', 'xpi-stage', addon_name, parts[1].lstrip('/'))) elif url_obj.scheme == 'file' and os.path.isabs(url_obj.path): path = url_obj.path if not os.path.isfile(path): @@ -620,12 +607,10 @@ class UrlFinder(object): self._final_mapping[url] = result return result - class LcovFileRewriter(object): # Class for partial parses of LCOV format and rewriting to resolve urls # and preprocessed file lines. - def __init__(self, chrome_map_path, appdir='dist/bin/browser/', - gredir='dist/bin/', extra_chrome_manifests=[]): + def __init__(self, chrome_map_path, appdir='dist/bin/browser/', gredir='dist/bin/', extra_chrome_manifests=[]): self.url_finder = UrlFinder(chrome_map_path, appdir, gredir, extra_chrome_manifests) self.pp_rewriter = RecordRewriter() @@ -646,11 +631,9 @@ class LcovFileRewriter(object): return None source_file, pp_info = res - # We can't assert that the file exists here, because we don't have the source - # checkout available on test machines. We can bring back this assertion when - # bug 1432287 is fixed. - # assert os.path.isfile(source_file), "Couldn't find mapped source file %s at %s!" % ( - # url, source_file) + # We can't assert that the file exists here, because we don't have the source checkout available + # on test machines. We can bring back this assertion when bug 1432287 is fixed. + # assert os.path.isfile(source_file), "Couldn't find mapped source file %s at %s!" % (url, source_file) found_valid[0] = True @@ -674,44 +657,28 @@ class LcovFileRewriter(object): def main(): - parser = ArgumentParser( - description="Given a set of gcov .info files produced " - "by spidermonkey's code coverage, re-maps file urls " - "back to source files and lines in preprocessed files " - "back to their original locations." - ) - parser.add_argument( - "--chrome-map-path", default="chrome-map.json", help="Path to the chrome-map.json file." - ) - parser.add_argument( - "--app-dir", - default="dist/bin/browser/", - help="Prefix of the appdir in use. This is used to map " - "urls starting with resource:///. It may differ by " - "app, but defaults to the valid value for firefox.", - ) - parser.add_argument( - "--gre-dir", - default="dist/bin/", - help="Prefix of the gre dir in use. This is used to map " - "urls starting with resource://gre. It may differ by " - "app, but defaults to the valid value for firefox.", - ) - parser.add_argument( - "--output-suffix", default=".out", help="The suffix to append to output files." - ) - parser.add_argument( - "--extra-chrome-manifests", - nargs='+', - help="Paths to files containing extra chrome registration.", - ) - parser.add_argument( - "--output-file", - default="", - help="The output file where the results are merged. Leave empty to make the rewriter not " - "merge files.", - ) - parser.add_argument("files", nargs='+', help="The set of files to process.") + parser = ArgumentParser(description="Given a set of gcov .info files produced " + "by spidermonkey's code coverage, re-maps file urls " + "back to source files and lines in preprocessed files " + "back to their original locations.") + parser.add_argument("--chrome-map-path", default="chrome-map.json", + help="Path to the chrome-map.json file.") + parser.add_argument("--app-dir", default="dist/bin/browser/", + help="Prefix of the appdir in use. This is used to map " + "urls starting with resource:///. It may differ by " + "app, but defaults to the valid value for firefox.") + parser.add_argument("--gre-dir", default="dist/bin/", + help="Prefix of the gre dir in use. This is used to map " + "urls starting with resource://gre. It may differ by " + "app, but defaults to the valid value for firefox.") + parser.add_argument("--output-suffix", default=".out", + help="The suffix to append to output files.") + parser.add_argument("--extra-chrome-manifests", nargs='+', + help="Paths to files containing extra chrome registration.") + parser.add_argument("--output-file", default="", + help="The output file where the results are merged. Leave empty to make the rewriter not merge files.") + parser.add_argument("files", nargs='+', + help="The set of files to process.") args = parser.parse_args() @@ -727,6 +694,5 @@ def main(): rewriter.rewrite_files(files, args.output_file, args.output_suffix) - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/codecoverage/manifest_handler.py b/python/mozbuild/mozbuild/codecoverage/manifest_handler.py index 63b3eab64079..1c49bb147855 100644 --- a/python/mozbuild/mozbuild/codecoverage/manifest_handler.py +++ b/python/mozbuild/mozbuild/codecoverage/manifest_handler.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from collections import defaultdict try: diff --git a/python/mozbuild/mozbuild/codecoverage/packager.py b/python/mozbuild/mozbuild/codecoverage/packager.py index e94e95716122..6e5c975c67ed 100644 --- a/python/mozbuild/mozbuild/codecoverage/packager.py +++ b/python/mozbuild/mozbuild/codecoverage/packager.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, print_function import argparse -import errno import sys import json import buildconfig @@ -18,7 +17,6 @@ from mozpack.manifests import ( ) import mozpack.path as mozpath - def describe_install_manifest(manifest, dest_dir): try: manifest = InstallManifest(manifest) @@ -77,6 +75,5 @@ def cli(args=sys.argv[1:]): return package_coverage_data(args.root, args.output_file) - if __name__ == '__main__': sys.exit(cli()) diff --git a/python/mozbuild/mozbuild/compilation/codecomplete.py b/python/mozbuild/mozbuild/compilation/codecomplete.py index 440d4d598a97..05583961a552 100644 --- a/python/mozbuild/mozbuild/compilation/codecomplete.py +++ b/python/mozbuild/mozbuild/compilation/codecomplete.py @@ -4,7 +4,9 @@ # This modules provides functionality for dealing with code completion. -from __future__ import absolute_import, print_function +from __future__ import absolute_import + +import os from mach.decorators import ( CommandArgument, @@ -24,9 +26,9 @@ class Introspection(MachCommandBase): """Instropection commands.""" @Command('compileflags', category='devenv', - description='Display the compilation flags for a given source file') + description='Display the compilation flags for a given source file') @CommandArgument('what', default=None, - help='Source file to display compilation flags for') + help='Source file to display compilation flags for') def compileflags(self, what): from mozbuild.util import resolve_target_to_make from mozbuild.compilation import util @@ -37,7 +39,7 @@ class Introspection(MachCommandBase): path_arg = self._wrap_path_argument(what) make_dir, make_target = resolve_target_to_make(self.topobjdir, - path_arg.relpath()) + path_arg.relpath()) if make_dir is None and make_target is None: return 1 diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py index 38ce376ec47f..bbb634f712aa 100644 --- a/python/mozbuild/mozbuild/compilation/database.py +++ b/python/mozbuild/mozbuild/compilation/database.py @@ -4,8 +4,6 @@ # This modules provides functionality for dealing with code completion. -from __future__ import absolute_import, print_function - import os import types @@ -15,8 +13,11 @@ from mozbuild.frontend.data import ( Sources, GeneratedSources, DirectoryTraversal, + Linkable, + LocalInclude, PerSourceFlag, VariablePassthru, + SimpleProgram, ) from mozbuild.shellutil import ( quote as shell_quote, @@ -175,8 +176,8 @@ class CompileDBBackend(CommonBackend): if canonical_suffix not in self.COMPILERS: return db = self._db.setdefault((objdir, filename, unified), - cenv.substs[self.COMPILERS[canonical_suffix]].split() + - ['-o', '/dev/null', '-c']) + cenv.substs[self.COMPILERS[canonical_suffix]].split() + + ['-o', '/dev/null', '-c']) reldir = reldir or mozpath.relpath(objdir, cenv.topobjdir) def append_var(name): diff --git a/python/mozbuild/mozbuild/compilation/util.py b/python/mozbuild/mozbuild/compilation/util.py index 1bcb9a3ec3cb..32ff2f8767ac 100644 --- a/python/mozbuild/mozbuild/compilation/util.py +++ b/python/mozbuild/mozbuild/compilation/util.py @@ -2,20 +2,17 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os - +from mozbuild import shellutil def check_top_objdir(topobjdir): top_make = os.path.join(topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been built yet. Please run ' - '|mach build| with no arguments.') + '|mach build| with no arguments.') return False return True - def get_build_vars(directory, cmd): build_vars = {} @@ -30,14 +27,13 @@ def get_build_vars(directory, cmd): try: old_logger = cmd.log_manager.replace_terminal_handler(None) cmd._run_make(directory=directory, target='showbuild', log=False, - print_directory=False, allow_parallel=False, silent=True, - line_handler=on_line) + print_directory=False, allow_parallel=False, silent=True, + line_handler=on_line) finally: cmd.log_manager.replace_terminal_handler(old_logger) return build_vars - def sanitize_cflags(flags): # We filter out -Xclang arguments as clang based tools typically choke on # passing these flags down to the clang driver. -Xclang tells the clang diff --git a/python/mozbuild/mozbuild/compilation/warnings.py b/python/mozbuild/mozbuild/compilation/warnings.py index fa786deef5c5..999189ea6ee5 100644 --- a/python/mozbuild/mozbuild/compilation/warnings.py +++ b/python/mozbuild/mozbuild/compilation/warnings.py @@ -4,7 +4,7 @@ # This modules provides functionality for dealing with compiler warnings. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import errno import json @@ -87,22 +87,22 @@ class CompilerWarning(dict): return func(self._cmpkey(), other._cmpkey()) def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) + return self._compare(other, lambda s,o: s == o) def __neq__(self, other): - return self._compare(other, lambda s, o: s != o) + return self._compare(other, lambda s,o: s != o) def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) + return self._compare(other, lambda s,o: s < o) def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) + return self._compare(other, lambda s,o: s <= o) def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) + return self._compare(other, lambda s,o: s > o) def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) + return self._compare(other, lambda s,o: s >= o) def __hash__(self): """Define so this can exist inside a set, etc.""" @@ -132,7 +132,6 @@ class WarningsDatabase(object): Callers should periodically prune old, invalid warnings from the database by calling prune(). A good time to do this is at the end of a build. """ - def __init__(self): """Create an empty database.""" self._files = {} @@ -305,7 +304,6 @@ class WarningsCollector(object): output from the compiler. Therefore, it can maintain state to parse multi-line warning messages. """ - def __init__(self, cb, objdir=None): """Initialize a new collector. diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py index 6da22f009f32..c5bd24728e27 100644 --- a/python/mozbuild/mozbuild/config_status.py +++ b/python/mozbuild/mozbuild/config_status.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, print_function import logging import os +import subprocess import sys import time @@ -79,14 +80,14 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, if 'CONFIG_FILES' in os.environ: raise Exception('Using the CONFIG_FILES environment variable is not ' - 'supported.') + 'supported.') if 'CONFIG_HEADERS' in os.environ: raise Exception('Using the CONFIG_HEADERS environment variable is not ' - 'supported.') + 'supported.') if not os.path.isabs(topsrcdir): raise Exception('topsrcdir must be defined as an absolute directory: ' - '%s' % topsrcdir) + '%s' % topsrcdir) default_backends = ['RecursiveMake'] default_backends = (substs or {}).get('BUILD_BACKENDS', ['RecursiveMake']) @@ -111,8 +112,8 @@ def config_status(topobjdir='.', topsrcdir='.', defines=None, topobjdir = os.path.abspath('.') env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines, - non_global_defines=non_global_defines, substs=substs, - source=source, mozconfig=mozconfig) + non_global_defines=non_global_defines, substs=substs, + source=source, mozconfig=mozconfig) with FileAvoidWrite(os.path.join(topobjdir, 'mozinfo.json')) as f: write_mozinfo(f, env, os.environ) diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py index 8e95b04d94b8..6b411bdc28a7 100644 --- a/python/mozbuild/mozbuild/configure/__init__.py +++ b/python/mozbuild/mozbuild/configure/__init__.py @@ -49,7 +49,6 @@ class ConfigureError(Exception): class SandboxDependsFunction(object): '''Sandbox-visible representation of @depends functions.''' - def __init__(self, unsandboxed): self._or = unsandboxed.__or__ self._and = unsandboxed.__and__ @@ -234,7 +233,6 @@ class CombinedDependsFunction(DependsFunction): def __ne__(self, other): return not self == other - class SandboxedGlobal(dict): '''Identifiable dict type for use as function global''' @@ -359,12 +357,10 @@ class ConfigureSandbox(dict): # that can't be converted to ascii. Make our log methods robust to this # by detecting the encoding that a producer is likely to have used. encoding = getpreferredencoding() - def wrapped_log_method(logger, key): method = getattr(logger, key) if not encoding: return method - def wrapped(*args, **kwargs): out_args = [ arg.decode(encoding) if isinstance(arg, str) else arg @@ -665,7 +661,7 @@ class ConfigureSandbox(dict): when = self._normalize_when(kwargs.get('when'), 'option') args = [self._resolve(arg) for arg in args] kwargs = {k: self._resolve(v) for k, v in kwargs.iteritems() - if k != 'when'} + if k != 'when'} option = Option(*args, **kwargs) if when: self._conditions[option] = when diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py index 5db8368fd9ef..c0caa9cc52f9 100644 --- a/python/mozbuild/mozbuild/configure/check_debug_ranges.py +++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py @@ -6,13 +6,12 @@ # to a given compilation unit. This is used as a helper to find a bug in some # versions of GNU ld. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import subprocess import sys import re - def get_range_for(compilation_unit, debug_info): '''Returns the range offset for a given compilation unit in a given debug_info.''' @@ -33,7 +32,6 @@ def get_range_for(compilation_unit, debug_info): ranges = nfo.rsplit(None, 1)[1] return None - def get_range_length(range, debug_ranges): '''Returns the number of items in the range starting at the given offset.''' @@ -44,9 +42,8 @@ def get_range_length(range, debug_ranges): length += 1 return length - def main(bin, compilation_unit): - p = subprocess.Popen(['objdump', '-W', bin], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = subprocess.Popen(['objdump', '-W', bin], stdout = subprocess.PIPE, stderr = subprocess.PIPE) (out, err) = p.communicate() sections = re.split('\n(Contents of the|The section) ', out) debug_info = [s for s in sections if s.startswith('.debug_info')] diff --git a/python/mozbuild/mozbuild/configure/libstdcxx.py b/python/mozbuild/mozbuild/configure/libstdcxx.py index 5f2e13d79879..b78d0cb44c5d 100755 --- a/python/mozbuild/mozbuild/configure/libstdcxx.py +++ b/python/mozbuild/mozbuild/configure/libstdcxx.py @@ -13,7 +13,7 @@ # will be used from shell, we just print the two assignments and evaluate # them from shell. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import subprocess @@ -21,14 +21,12 @@ import re re_for_ld = re.compile('.*\((.*)\).*') - def parse_readelf_line(x): """Return the version from a readelf line that looks like: 0x00ec: Rev: 1 Flags: none Index: 8 Cnt: 2 Name: GLIBCXX_3.4.6 """ return x.split(':')[-1].split('_')[-1].strip() - def parse_ld_line(x): """Parse a line from the output of ld -t. The output of gold is just the full path, gnu ld prints "-lstdc++ (path)". @@ -38,13 +36,11 @@ def parse_ld_line(x): return t.groups()[0].strip() return x.strip() - def split_ver(v): """Covert the string '1.2.3' into the list [1,2,3] """ return [int(x) for x in v.split('.')] - def cmp_ver(a, b): """Compare versions in the form 'a.b.c' """ @@ -53,19 +49,17 @@ def cmp_ver(a, b): return i - j return 0 - def encode_ver(v): """Encode the version as a single number. """ t = split_ver(v) return t[0] << 16 | t[1] << 8 | t[2] - def find_version(args): """Given a base command line for a compiler, find the version of the libstdc++ it uses. """ - args += ['-shared', '-Wl,-t'] + args += ['-shared', '-Wl,-t'] p = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) candidates = [x for x in p.stdout if 'libstdc++.so' in x] candidates = [x for x in candidates if 'skipping incompatible' not in x] @@ -83,10 +77,9 @@ candidates: p = subprocess.Popen(['readelf', '-V', libstdcxx], stdout=subprocess.PIPE) versions = [parse_readelf_line(x) for x in p.stdout.readlines() if 'Name: GLIBCXX' in x] - last_version = sorted(versions, cmp=cmp_ver)[-1] + last_version = sorted(versions, cmp = cmp_ver)[-1] return (last_version, encode_ver(last_version)) - if __name__ == '__main__': """Given the value of environment variable CXX or HOST_CXX, find the version of the libstdc++ it uses. diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py index 9afe7a78fc6f..1a6f5527ff1e 100644 --- a/python/mozbuild/mozbuild/configure/lint.py +++ b/python/mozbuild/mozbuild/configure/lint.py @@ -237,6 +237,7 @@ class LintSandbox(ConfigureSandbox): name, default)) self._raise_from(e, frame.f_back if frame else None) + def _check_help_for_option_with_func_default(self, option, *args, **kwargs): default = kwargs['default'] @@ -284,7 +285,6 @@ class LintSandbox(ConfigureSandbox): def imports_impl(self, _import, _from=None, _as=None): wrapper = super(LintSandbox, self).imports_impl(_import, _from=_from, _as=_as) - def decorator(func): self._has_imports.add(func) return wrapper(func) diff --git a/python/mozbuild/mozbuild/configure/lint_util.py b/python/mozbuild/mozbuild/configure/lint_util.py index 52e33920ee53..4d6fea3f63b1 100644 --- a/python/mozbuild/mozbuild/configure/lint_util.py +++ b/python/mozbuild/mozbuild/configure/lint_util.py @@ -46,7 +46,7 @@ def disassemble_as_iter(co): c = code[i] op = ord(c) opname = dis.opname[op] - i += 1 + i += 1; if op >= dis.HAVE_ARGUMENT: arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg extended_arg = 0 diff --git a/python/mozbuild/mozbuild/configure/options.py b/python/mozbuild/mozbuild/configure/options.py index 8a403272c31c..8022c304a2b6 100644 --- a/python/mozbuild/mozbuild/configure/options.py +++ b/python/mozbuild/mozbuild/configure/options.py @@ -106,7 +106,6 @@ class PositiveOptionValue(OptionValue): in the form of a tuple for when values are given to the option (in the form --option=value[,value2...]. ''' - def __nonzero__(self): return True @@ -425,7 +424,6 @@ class CommandLineHelper(object): Extra options can be added afterwards through API calls. For those, conflicting values will raise an exception. ''' - def __init__(self, environ=os.environ, argv=sys.argv): self._environ = dict(environ) self._args = OrderedDict() diff --git a/python/mozbuild/mozbuild/configure/util.py b/python/mozbuild/mozbuild/configure/util.py index 8141f81d00c8..9d8b2eb0ec66 100644 --- a/python/mozbuild/mozbuild/configure/util.py +++ b/python/mozbuild/mozbuild/configure/util.py @@ -14,7 +14,6 @@ from collections import deque from contextlib import contextmanager from distutils.version import LooseVersion - def getpreferredencoding(): # locale._parse_localename makes locale.getpreferredencoding # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or @@ -30,7 +29,6 @@ def getpreferredencoding(): encoding = 'utf-8' return encoding - class Version(LooseVersion): '''A simple subclass of distutils.version.LooseVersion. Adds attributes for `major`, `minor`, `patch` for the first three @@ -42,14 +40,13 @@ class Version(LooseVersion): v.minor == 2 v.patch == 0 ''' - def __init__(self, version): # Can't use super, LooseVersion's base class is not a new-style class. LooseVersion.__init__(self, version) # Take the first three integer components, stopping at the first # non-integer and padding the rest with zeroes. (self.major, self.minor, self.patch) = list(itertools.chain( - itertools.takewhile(lambda x: isinstance(x, int), self.version), + itertools.takewhile(lambda x:isinstance(x, int), self.version), (0, 0, 0)))[:3] def __cmp__(self, other): @@ -74,7 +71,6 @@ class ConfigureOutputHandler(logging.Handler): printed out. This feature is only enabled under the `queue_debug` context manager. ''' - def __init__(self, stdout=sys.stdout, stderr=sys.stderr, maxlen=20): super(ConfigureOutputHandler, self).__init__() @@ -155,7 +151,7 @@ class ConfigureOutputHandler(logging.Handler): stream.flush() except (KeyboardInterrupt, SystemExit, IOError): raise - except Exception: + except: self.handleError(record) @contextmanager @@ -197,7 +193,6 @@ class LineIO(object): '''File-like class that sends each line of the written data to a callback (without carriage returns). ''' - def __init__(self, callback, errors='strict'): self._callback = callback self._buf = '' diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py index 9ff3c7ea4ce4..12033358f993 100644 --- a/python/mozbuild/mozbuild/controller/building.py +++ b/python/mozbuild/mozbuild/controller/building.py @@ -2,10 +2,11 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import errno import getpass +import glob import io import json import logging @@ -78,7 +79,7 @@ Preferences. INSTALL_TESTS_CLOBBER = ''.join([TextWrapper().fill(line) + '\n' for line in - ''' +''' The build system was unable to install tests because the CLOBBER file has \ been updated. This means if you edited any test files, your changes may not \ be picked up until a full/clobber build is performed. @@ -106,7 +107,7 @@ this is a clobber bug and not due to local changes. BuildOutputResult = namedtuple('BuildOutputResult', - ('warning', 'state_changed', 'message')) + ('warning', 'state_changed', 'message')) class TierStatus(object): @@ -172,9 +173,9 @@ class TierStatus(object): def add_resources_to_dict(self, entry, start=None, end=None, phase=None): """Helper function to append resource information to a dict.""" cpu_percent = self.resources.aggregate_cpu_percent(start=start, - end=end, phase=phase, per_cpu=False) + end=end, phase=phase, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(start=start, end=end, - phase=phase, per_cpu=False) + phase=phase, per_cpu=False) io = self.resources.aggregate_io(start=start, end=end, phase=phase) if cpu_percent is None: @@ -304,7 +305,7 @@ class BuildMonitor(MozbuildObject): try: warning = self._warnings_collector.process_line(line) message = line - except Exception: + except: pass return BuildOutputResult(warning, False, message) @@ -338,8 +339,8 @@ class BuildMonitor(MozbuildObject): json.dump(self.resources.as_dict(), fh, indent=2) except Exception as e: self.log(logging.WARNING, 'build_resources_error', - {'msg': str(e)}, - 'Exception when writing resource usage file: {msg}') + {'msg': str(e)}, + 'Exception when writing resource usage file: {msg}') def _get_finder_cpu_usage(self): """Obtain the CPU usage of the Finder app on OS X. @@ -437,9 +438,9 @@ class BuildMonitor(MozbuildObject): return None cpu_percent = self.resources.aggregate_cpu_percent(phase=None, - per_cpu=False) + per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(phase=None, - per_cpu=False) + per_cpu=False) io = self.resources.aggregate_io(phase=None) o = dict( @@ -461,9 +462,9 @@ class BuildMonitor(MozbuildObject): for usage in self.resources.range_usage(): cpu_percent = self.resources.aggregate_cpu_percent(usage.start, - usage.end, per_cpu=False) + usage.end, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(usage.start, - usage.end, per_cpu=False) + usage.end, per_cpu=False) entry = dict( start=usage.start, @@ -473,10 +474,11 @@ class BuildMonitor(MozbuildObject): ) self.tiers.add_resources_to_dict(entry, start=usage.start, - end=usage.end) + end=usage.end) o['resources'].append(entry) + # If the imports for this file ran before the in-tree virtualenv # was bootstrapped (for instance, for a clobber build in automation), # psutil might not be available. @@ -522,8 +524,8 @@ class BuildMonitor(MozbuildObject): sin /= 1048576 sout /= 1048576 self.log(logging.WARNING, 'swap_activity', - {'sin': sin, 'sout': sout}, - 'Swap in/out (MB): {sin}/{sout}') + {'sin': sin, 'sout': sout}, + 'Swap in/out (MB): {sin}/{sout}') def ccache_stats(self): ccache_stats = None @@ -546,7 +548,6 @@ class TerminalLoggingHandler(logging.Handler): This class should probably live elsewhere, like the mach core. Consider this a proving ground for its usefulness. """ - def __init__(self): logging.Handler.__init__(self) @@ -682,6 +683,7 @@ class BuildOutputManager(OutputManager): # collection child process hasn't been told to stop. self.monitor.stop_resource_recording() + def on_line(self, line): warning, state_changed, message = self.monitor.on_line(line) @@ -742,7 +744,7 @@ class StaticAnalysisOutputManager(OutputManager): if warning: self.log(logging.INFO, 'compiler_warning', warning, - 'Warning: {flag} in {filename}: {message}') + 'Warning: {flag} in {filename}: {message}') if relevant: self.log(logging.INFO, 'build_output', {'line': line}, '{line}') @@ -784,7 +786,7 @@ class CCacheStats(object): STATS_KEYS = [ # (key, description) # Refer to stats.c in ccache project for all the descriptions. - ('stats_zeroed', 'stats zero time'), # Old name prior to ccache 3.4 + ('stats_zeroed', 'stats zero time'), # Old name prior to ccache 3.4 ('stats_zeroed', 'stats zeroed'), ('stats_updated', 'stats updated'), ('cache_hit_direct', 'cache hit (direct)'), @@ -901,10 +903,7 @@ class CCacheStats(object): return int(numeric * unit) def hit_rate_message(self): - return ('ccache (direct) hit rate: {:.1%}; (preprocessed) hit rate: {:.1%};' - ' miss rate: {:.1%}'.format( - *self.hit_rates() - )) + return 'ccache (direct) hit rate: {:.1%}; (preprocessed) hit rate: {:.1%}; miss rate: {:.1%}'.format(*self.hit_rates()) def hit_rates(self): direct = self._values['cache_hit_direct'] @@ -1006,11 +1005,11 @@ class BuildDriver(MozbuildObject): if directory is not None and not what: print('Can only use -C/--directory with an explicit target ' - 'name.') + 'name.') return 1 if directory is not None: - disable_extra_make_dependencies = True + disable_extra_make_dependencies=True directory = mozpath.normsep(directory) if directory.startswith('/'): directory = directory[1:] @@ -1105,7 +1104,7 @@ class BuildDriver(MozbuildObject): else: make_dir, make_target = \ resolve_target_to_make(self.topobjdir, - path_arg.relpath()) + path_arg.relpath()) if make_dir is None and make_target is None: return 1 @@ -1146,12 +1145,10 @@ class BuildDriver(MozbuildObject): # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. no_build_status = b'1' if make_dir is not None else b'' - status = self._run_make( - directory=make_dir, target=make_target, + status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, - append_env={ - b'NO_BUILDSTATUS_MESSAGES': no_build_status}, + append_env={b'NO_BUILDSTATUS_MESSAGES': no_build_status}, keep_going=keep_going) if status != 0: @@ -1207,8 +1204,8 @@ class BuildDriver(MozbuildObject): # in these directories. pathToThirdparty = os.path.join(self.topsrcdir, "tools", - "rewriting", - "ThirdPartyPaths.txt") + "rewriting", + "ThirdPartyPaths.txt") if os.path.exists(pathToThirdparty): with open(pathToThirdparty) as f: @@ -1287,8 +1284,7 @@ class BuildDriver(MozbuildObject): long_build = monitor.elapsed > 600 if long_build: - output.on_line( - 'We know it took a while, but your build finally finished successfully!') + output.on_line('We know it took a while, but your build finally finished successfully!') else: output.on_line('Your build was successful!') @@ -1302,10 +1298,8 @@ class BuildDriver(MozbuildObject): print('To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): - print( - 'For more information on what to do now, see ' - 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox' # noqa - ) + print('For more information on what to do now, see ' + 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) @@ -1331,14 +1325,14 @@ class BuildDriver(MozbuildObject): # Only print build status messages when we have an active # monitor. if not buildstatus_messages: - append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1' + append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1' status = self._run_client_mk(target='configure', line_handler=line_handler, append_env=append_env) if not status: print('Configure complete!') - print('Be sure to run |mach build| to pick up any changes') + print('Be sure to run |mach build| to pick up any changes'); return status diff --git a/python/mozbuild/mozbuild/controller/clobber.py b/python/mozbuild/mozbuild/controller/clobber.py index da89b460a75d..f5fd08e2e129 100644 --- a/python/mozbuild/mozbuild/controller/clobber.py +++ b/python/mozbuild/mozbuild/controller/clobber.py @@ -16,7 +16,7 @@ from textwrap import TextWrapper CLOBBER_MESSAGE = ''.join([TextWrapper().fill(line) + '\n' for line in - ''' +''' The CLOBBER file has been updated, indicating that an incremental build since \ your last build will probably not work. A full/clobber build is required. @@ -39,7 +39,6 @@ Well, are ya? -- you can ignore this clobber requirement by running: $ touch {clobber_file} '''.splitlines()]) - class Clobberer(object): def __init__(self, topsrcdir, topobjdir): """Create a new object to manage clobbering the tree. @@ -70,7 +69,7 @@ class Clobberer(object): # Object directory clobber older than current is fine. if os.path.getmtime(self.src_clobber) <= \ - os.path.getmtime(self.obj_clobber): + os.path.getmtime(self.obj_clobber): return False @@ -95,7 +94,7 @@ class Clobberer(object): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) return p.wait() == 1 and p.stdout.read().startswith('winrm') - except Exception: + except: return False def remove_objdir(self, full=True): @@ -183,8 +182,8 @@ class Clobberer(object): if not allow_auto: return True, False, \ self._message('Automatic clobbering is not enabled\n' - ' (add "mk_add_options AUTOCLOBBER=1" to your ' - 'mozconfig).') + ' (add "mk_add_options AUTOCLOBBER=1" to your ' + 'mozconfig).') if cwd.startswith(self.topobjdir) and cwd != self.topobjdir: return True, False, self._message( @@ -205,4 +204,4 @@ class Clobberer(object): lines = [' ' + line for line in self.clobber_cause()] return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines), - no_reason=' ' + reason, clobber_file=self.obj_clobber) + no_reason=' ' + reason, clobber_file=self.obj_clobber) diff --git a/python/mozbuild/mozbuild/doctor.py b/python/mozbuild/mozbuild/doctor.py index c5bae1f8f55e..8b8b6678c84e 100644 --- a/python/mozbuild/mozbuild/doctor.py +++ b/python/mozbuild/mozbuild/doctor.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import subprocess @@ -34,7 +34,6 @@ hour. Backup programs that rely on this feature may be affected. https://technet.microsoft.com/en-us/library/cc785435.aspx ''' - class Doctor(object): def __init__(self, srcdir, objdir, fix): self.srcdir = mozpath.normpath(srcdir) @@ -70,7 +69,7 @@ class Doctor(object): denied = True if denied: print('run "mach doctor --fix" AS ADMIN to re-attempt fixing your system') - elif False and fixable: # elif fixable: # 'and fixable' avoids flake8 error + elif False: # elif fixable: print('run "mach doctor --fix" as admin to attempt fixing your system') return int(not good) @@ -203,7 +202,7 @@ class Doctor(object): status = 'GOOD' desc = 'lastaccess disabled systemwide' elif disablelastaccess == 0: - if False: # if self.fix: + if False: # if self.fix: choice = self.prompt_bool(DISABLE_LASTACCESS_WIN) if not choice: return {'status': 'BAD, NOT FIXED', diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py index a84898e602de..972ff23299d0 100644 --- a/python/mozbuild/mozbuild/dotproperties.py +++ b/python/mozbuild/mozbuild/dotproperties.py @@ -5,7 +5,7 @@ # This file contains utility functions for reading .properties files, like # region.properties. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import codecs import re @@ -16,7 +16,6 @@ if sys.version_info[0] == 3: else: str_type = basestring - class DotProperties: r'''A thin representation of a key=value .properties file.''' @@ -78,7 +77,7 @@ class DotProperties: if k.startswith(prefix) and '.' not in k[len(prefix):]) for required_key in required_keys: - if required_key not in D: + if not required_key in D: raise ValueError('Required key %s not present' % required_key) return D diff --git a/python/mozbuild/mozbuild/export_telemetry_schema.py b/python/mozbuild/mozbuild/export_telemetry_schema.py index 3fd844f70e87..9dd5ee5ba074 100644 --- a/python/mozbuild/mozbuild/export_telemetry_schema.py +++ b/python/mozbuild/mozbuild/export_telemetry_schema.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import print_function, unicode_literals """ This script converts the build system telemetry schema from voluptuous format to json-schema. diff --git a/python/mozbuild/mozbuild/faster_daemon.py b/python/mozbuild/mozbuild/faster_daemon.py index 74c9de4cb7af..ea9a32340220 100644 --- a/python/mozbuild/mozbuild/faster_daemon.py +++ b/python/mozbuild/mozbuild/faster_daemon.py @@ -108,8 +108,8 @@ class Daemon(object): ['name', '.hg', 'wholename'], ['dirname', '.git'], ['name', '.git', 'wholename'], - ], ], + ], ], 'fields': ['name'], } @@ -204,7 +204,7 @@ class Daemon(object): while True: try: - self.client.receive() + _watch_result = self.client.receive() changed = self.changed_files() if not changed: @@ -228,7 +228,7 @@ class Daemon(object): except pywatchman.SocketTimeout: # Let's check to see if we're still functional. - self.client.query('version') + _version = self.client.query('version') except pywatchman.CommandError as e: # Abstract away pywatchman errors. diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py index 659ab1c0a2c3..2e3242937616 100644 --- a/python/mozbuild/mozbuild/frontend/context.py +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -14,7 +14,7 @@ If you are looking for the absolute authority on what moz.build files can contain, you've come to the right place. """ -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import os @@ -89,7 +89,6 @@ class Context(KeyedDefaultDict): config is the ConfigEnvironment for this context. """ - def __init__(self, allowed_variables={}, config=None, finder=None): self._allowed_variables = allowed_variables self.main_path = None @@ -164,7 +163,7 @@ class Context(KeyedDefaultDict): @memoize def _srcdir(self, path): return mozpath.join(self.config.topsrcdir, - self._relsrcdir(path)).rstrip('/') + self._relsrcdir(path)).rstrip('/') @property def srcdir(self): @@ -183,7 +182,7 @@ class Context(KeyedDefaultDict): def relobjdir(self): assert self.main_path return mozpath.relpath(mozpath.dirname(self.main_path), - self.config.topsrcdir) + self.config.topsrcdir) def _factory(self, key): """Function called when requesting a missing key.""" @@ -270,7 +269,6 @@ class SubContext(Context, ContextDerivedValue): Sub-contexts inherit paths and other relevant state from the parent context. """ - def __init__(self, parent): assert isinstance(parent, Context) @@ -572,7 +570,6 @@ class PathMeta(type): cls = SourcePath return super(PathMeta, cls).__call__(context, value) - class Path(ContextDerivedValue, unicode): """Stores and resolves a source path relative to a given context @@ -639,7 +636,6 @@ class Path(ContextDerivedValue, unicode): class SourcePath(Path): """Like Path, but limited to paths in the source directory.""" - def __init__(self, context, value): if value.startswith('!'): raise ValueError('Object directory paths are not allowed') @@ -680,7 +676,6 @@ class RenamedSourcePath(SourcePath): This class is not meant to be exposed to moz.build sandboxes as of now, and is not supported by the RecursiveMake backend. """ - def __init__(self, context, value): assert isinstance(value, tuple) source, self._target_basename = value @@ -693,14 +688,13 @@ class RenamedSourcePath(SourcePath): class ObjDirPath(Path): """Like Path, but limited to paths in the object directory.""" - def __init__(self, context, value=None): if not value.startswith('!'): raise ValueError('Object directory paths must start with ! prefix') super(ObjDirPath, self).__init__(context, value) if value.startswith('!/'): - path = mozpath.join(context.config.topobjdir, value[2:]) + path = mozpath.join(context.config.topobjdir,value[2:]) else: path = mozpath.join(context.objdir, value[1:]) self.full_path = mozpath.normpath(path) @@ -708,7 +702,6 @@ class ObjDirPath(Path): class AbsolutePath(Path): """Like Path, but allows arbitrary paths outside the source and object directories.""" - def __init__(self, context, value=None): if not value.startswith('%'): raise ValueError('Absolute paths must start with % prefix') @@ -724,7 +717,6 @@ def ContextDerivedTypedList(klass, base_class=List): """Specialized TypedList for use with ContextDerivedValue types. """ assert issubclass(klass, ContextDerivedValue) - class _TypedList(ContextDerivedValue, TypedList(klass, base_class)): def __init__(self, context, iterable=[], **kwargs): self.context = context @@ -737,7 +729,6 @@ def ContextDerivedTypedList(klass, base_class=List): return _TypedList - @memoize def ContextDerivedTypedListWithItems(type, base_class=List): """Specialized TypedList for use with ContextDerivedValue types. @@ -871,7 +862,6 @@ def ContextDerivedTypedHierarchicalStringList(type): return _TypedListWithItems - def OrderedPathListWithAction(action): """Returns a class which behaves as a StrictOrderingOnAppendList, but invokes the given callable with each input and a context as it is @@ -881,7 +871,7 @@ def OrderedPathListWithAction(action): filesystem-reading mode. """ class _OrderedListWithAction(ContextDerivedTypedList(SourcePath, - StrictOrderingOnAppendListWithAction)): + StrictOrderingOnAppendListWithAction)): def __init__(self, context, *args): def _action(item): return item, action(context, item) @@ -889,7 +879,6 @@ def OrderedPathListWithAction(action): return _OrderedListWithAction - def TypedListWithAction(typ, action): """Returns a class which behaves as a TypedList with the provided type, but invokes the given given callable with each input and a context as it is @@ -905,7 +894,6 @@ def TypedListWithAction(typ, action): super(_TypedListWithAction, self).__init__(action=_action, *args) return _TypedListWithAction - ManifestparserManifestList = OrderedPathListWithAction(read_manifestparser_manifest) ReftestManifestList = OrderedPathListWithAction(read_reftest_manifest) @@ -917,7 +905,7 @@ DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList), ('tags', OrderedStringList), ('flavors', OrderedTestFlavorList)) BugzillaComponent = TypedNamedTuple('BugzillaComponent', - [('product', unicode), ('component', unicode)]) + [('product', unicode), ('component', unicode)]) SchedulingComponents = ContextDerivedTypedRecord( ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)), ('exclusive', TypedList(unicode, StrictOrderingOnAppendList))) @@ -977,14 +965,14 @@ class Files(SubContext): VARIABLES = { 'BUG_COMPONENT': (BugzillaComponent, tuple, - """The bug component that tracks changes to these files. + """The bug component that tracks changes to these files. Values are a 2-tuple of unicode describing the Bugzilla product and component. e.g. ``('Firefox Build System', 'General')``. """), 'FINAL': (bool, bool, - """Mark variable assignments as finalized. + """Mark variable assignments as finalized. During normal processing, values from newer Files contexts overwrite previously set values. Last write wins. This behavior is @@ -998,7 +986,7 @@ class Files(SubContext): See :ref:`mozbuild_files_metadata_finalizing` for more info. """), 'IMPACTED_TESTS': (DependentTestsEntry, list, - """File patterns, tags, and flavors for tests relevant to these files. + """File patterns, tags, and flavors for tests relevant to these files. Maps source files to the tests potentially impacted by those files. Tests can be specified by file pattern, tag, or flavor. @@ -1046,7 +1034,7 @@ class Files(SubContext): any plain mochitest. """), 'SCHEDULES': (Schedules, list, - """Maps source files to the CI tasks that should be scheduled when + """Maps source files to the CI tasks that should be scheduled when they change. The tasks are grouped by named components, and those names appear again in the taskgraph configuration `($topsrcdir/taskgraph/). @@ -1201,105 +1189,66 @@ SUBCONTEXTS = {cls.__name__: cls for cls in SUBCONTEXTS} # (storage_type, input_types, docs) VARIABLES = { - 'SOURCES': ( - ContextDerivedTypedListWithItems( - Path, - StrictOrderingOnAppendListWithFlagsFactory( - {'no_pgo': bool, - 'flags': List, - 'pgo_generate_only': bool - } - ) - ), - list, + 'SOURCES': (ContextDerivedTypedListWithItems(Path, StrictOrderingOnAppendListWithFlagsFactory({'no_pgo': bool, 'flags': List, 'pgo_generate_only': bool})), list, """Source code files. This variable contains a list of source code files to compile. Accepts assembler, C, C++, Objective C/C++. - """ - ), + """), - 'FILES_PER_UNIFIED_FILE': ( - int, - int, + 'FILES_PER_UNIFIED_FILE': (int, int, """The number of source files to compile into each unified source file. - """ - ), + """), - 'IS_RUST_LIBRARY': ( - bool, - bool, + 'IS_RUST_LIBRARY': (bool, bool, """Whether the current library defined by this moz.build is built by Rust. The library defined by this moz.build should have a build definition in a Cargo.toml file that exists in this moz.build's directory. - """ - ), + """), - 'RUST_LIBRARY_FEATURES': ( - List, - list, + 'RUST_LIBRARY_FEATURES': (List, list, """Cargo features to activate for this library. This variable should not be used directly; you should be using the RustLibrary template instead. - """ - ), + """), - 'RUST_LIBRARY_TARGET_DIR': ( - unicode, - unicode, + 'RUST_LIBRARY_TARGET_DIR': (unicode, unicode, """Where CARGO_TARGET_DIR should point when compiling this library. If not set, it defaults to the current objdir. It should be a relative path to the current objdir; absolute paths should not be used. This variable should not be used directly; you should be using the RustLibrary template instead. - """ - ), + """), - 'HOST_RUST_LIBRARY_FEATURES': ( - List, - list, + 'HOST_RUST_LIBRARY_FEATURES': (List, list, """Cargo features to activate for this host library. This variable should not be used directly; you should be using the HostRustLibrary template instead. - """ - ), + """), - 'RUST_TESTS': ( - TypedList(unicode), - list, + 'RUST_TESTS': (TypedList(unicode), list, """Names of Rust tests to build and run via `cargo test`. """), - 'RUST_TEST_FEATURES': ( - TypedList(unicode), - list, + 'RUST_TEST_FEATURES': (TypedList(unicode), list, """Cargo features to activate for RUST_TESTS. - """ - ), + """), - 'UNIFIED_SOURCES': ( - ContextDerivedTypedList( - SourcePath, - StrictOrderingOnAppendList - ), - list, + 'UNIFIED_SOURCES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, """Source code files that can be compiled together. This variable contains a list of source code files to compile, that can be concatenated all together and built as a single source file. This can help make the build faster and reduce the debug info size. - """ - ), + """), - 'GENERATED_FILES': ( - GeneratedFilesList, - list, + 'GENERATED_FILES': (GeneratedFilesList, list, """Generic generated files. This variable contains a list of files for the build system to @@ -1348,12 +1297,9 @@ VARIABLES = { build, regardless of whether it is stale. This is special to the RecursiveMake backend and intended for special situations only (e.g., localization). Please consult a build peer before using ``force``. - """ - ), + """), - 'DEFINES': ( - InitializedDefines, - dict, + 'DEFINES': (InitializedDefines, dict, """Dictionary of compiler defines to declare. These are passed in to the compiler as ``-Dkey='value'`` for string @@ -1379,22 +1325,16 @@ VARIABLES = { 'MOZ_EXTENSIONS_DB_SCHEMA': 15, 'DLL_SUFFIX': '".so"', }) - """ - ), + """), - 'DELAYLOAD_DLLS': ( - List, - list, + 'DELAYLOAD_DLLS': (List, list, """Delay-loaded DLLs. This variable contains a list of DLL files which the module being linked should load lazily. This only has an effect when building with MSVC. - """ - ), + """), - 'DIRS': ( - ContextDerivedTypedList(SourcePath), - list, + 'DIRS': (ContextDerivedTypedList(SourcePath), list, """Child directories to descend into looking for build frontend files. This works similarly to the ``DIRS`` variable in make files. Each str @@ -1406,12 +1346,9 @@ VARIABLES = { Values are relative paths. They can be multiple directory levels above or below. Use ``..`` for parent directories and ``/`` for path delimiters. - """ - ), + """), - 'HAS_MISC_RULE': ( - bool, - bool, + 'HAS_MISC_RULE': (bool, bool, """Whether this directory should be traversed in the ``misc`` tier. Many ``libs`` rules still exist in Makefile.in files. We highly prefer @@ -1425,11 +1362,10 @@ VARIABLES = { Please note that converting ``libs`` rules to the ``misc`` tier must be done with care, as there are many implicit dependencies that can break the build in subtle ways. - """ - ), + """), 'FINAL_TARGET_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be installed into the application directory. + """List of files to be installed into the application directory. ``FINAL_TARGET_FILES`` will copy (or symlink, if the platform supports it) the contents of its files to the directory specified by @@ -1444,11 +1380,11 @@ VARIABLES = { """), 'FINAL_TARGET_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """Like ``FINAL_TARGET_FILES``, with preprocessing. + """Like ``FINAL_TARGET_FILES``, with preprocessing. """), 'LOCALIZED_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of locale-dependent files to be installed into the application + """List of locale-dependent files to be installed into the application directory. This functions similarly to ``FINAL_TARGET_FILES``, but the files are @@ -1490,14 +1426,14 @@ VARIABLES = { """), 'LOCALIZED_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """Like ``LOCALIZED_FILES``, with preprocessing. + """Like ``LOCALIZED_FILES``, with preprocessing. Note that the ``AB_CD`` define is available and expands to the current locale being packaged, as with preprocessed entries in jar manifests. """), 'LOCALIZED_GENERATED_FILES': (GeneratedFilesList, list, - """Like ``GENERATED_FILES``, but for files whose content varies based on the locale in use. + """Like ``GENERATED_FILES``, but for files whose content varies based on the locale in use. For simple cases of text substitution, prefer ``LOCALIZED_PP_FILES``. @@ -1521,7 +1457,7 @@ VARIABLES = { """), 'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be installed anywhere in the objdir. Use sparingly. + """List of files to be installed anywhere in the objdir. Use sparingly. ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying anywhere in the object directory. This is intended for various one-off @@ -1530,11 +1466,11 @@ VARIABLES = { """), 'OBJDIR_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly. + """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly. """), 'FINAL_LIBRARY': (unicode, unicode, - """Library in which the objects of the current directory will be linked. + """Library in which the objects of the current directory will be linked. This variable contains the name of a library, defined elsewhere with ``LIBRARY_NAME``, in which the objects of the current directory will be @@ -1542,7 +1478,7 @@ VARIABLES = { """), 'CPP_UNIT_TESTS': (StrictOrderingOnAppendList, list, - """Compile a list of C++ unit test names. + """Compile a list of C++ unit test names. Each name in this variable corresponds to an executable built from the corresponding source file with the same base name. @@ -1553,40 +1489,40 @@ VARIABLES = { """), 'FORCE_SHARED_LIB': (bool, bool, - """Whether the library in this directory is a shared library. + """Whether the library in this directory is a shared library. """), 'FORCE_STATIC_LIB': (bool, bool, - """Whether the library in this directory is a static library. + """Whether the library in this directory is a static library. """), 'USE_STATIC_LIBS': (bool, bool, - """Whether the code in this directory is a built against the static + """Whether the code in this directory is a built against the static runtime library. This variable only has an effect when building with MSVC. """), 'HOST_SOURCES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list, - """Source code files to compile with the host compiler. + """Source code files to compile with the host compiler. This variable contains a list of source code files to compile. with the host compiler. """), 'HOST_LIBRARY_NAME': (unicode, unicode, - """Name of target library generated when cross compiling. + """Name of target library generated when cross compiling. """), 'LIBRARY_DEFINES': (OrderedDict, dict, - """Dictionary of compiler defines to declare for the entire library. + """Dictionary of compiler defines to declare for the entire library. This variable works like DEFINES, except that declarations apply to all libraries that link into this library via FINAL_LIBRARY. """), 'LIBRARY_NAME': (unicode, unicode, - """The code name of the library generated for a directory. + """The code name of the library generated for a directory. By default STATIC_LIBRARY_NAME and SHARED_LIBRARY_NAME take this name. In ``example/components/moz.build``,:: @@ -1598,89 +1534,89 @@ VARIABLES = { """), 'SHARED_LIBRARY_NAME': (unicode, unicode, - """The name of the static library generated for a directory, if it needs to + """The name of the static library generated for a directory, if it needs to differ from the library code name. Implies FORCE_SHARED_LIB. """), 'SHARED_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode, - """The output category for this context's shared library. If set this will + """The output category for this context's shared library. If set this will correspond to the build command that will build this shared library, and the library will not be built as part of the default build. """), 'RUST_LIBRARY_OUTPUT_CATEGORY': (unicode, unicode, - """The output category for this context's rust library. If set this will + """The output category for this context's rust library. If set this will correspond to the build command that will build this rust library, and the library will not be built as part of the default build. """), 'IS_FRAMEWORK': (bool, bool, - """Whether the library to build should be built as a framework on OSX. + """Whether the library to build should be built as a framework on OSX. This implies the name of the library won't be prefixed nor suffixed. Implies FORCE_SHARED_LIB. """), 'STATIC_LIBRARY_NAME': (unicode, unicode, - """The name of the static library generated for a directory, if it needs to + """The name of the static library generated for a directory, if it needs to differ from the library code name. Implies FORCE_STATIC_LIB. """), 'USE_LIBS': (StrictOrderingOnAppendList, list, - """List of libraries to link to programs and libraries. + """List of libraries to link to programs and libraries. """), 'HOST_USE_LIBS': (StrictOrderingOnAppendList, list, - """List of libraries to link to host programs and libraries. + """List of libraries to link to host programs and libraries. """), 'HOST_OS_LIBS': (List, list, - """List of system libraries for host programs and libraries. + """List of system libraries for host programs and libraries. """), 'LOCAL_INCLUDES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list, - """Additional directories to be searched for include files by the compiler. + """Additional directories to be searched for include files by the compiler. """), 'NO_PGO': (bool, bool, - """Whether profile-guided optimization is disable in this directory. + """Whether profile-guided optimization is disable in this directory. """), 'OS_LIBS': (List, list, - """System link libraries. + """System link libraries. This variable contains a list of system libaries to link against. """), 'RCFILE': (unicode, unicode, - """The program .rc file. + """The program .rc file. This variable can only be used on Windows. """), 'RESFILE': (unicode, unicode, - """The program .res file. + """The program .res file. This variable can only be used on Windows. """), 'RCINCLUDE': (unicode, unicode, - """The resource script file to be included in the default .res file. + """The resource script file to be included in the default .res file. This variable can only be used on Windows. """), 'DEFFILE': (Path, unicode, - """The program .def (module definition) file. + """The program .def (module definition) file. This variable can only be used on Windows. """), 'SYMBOLS_FILE': (Path, unicode, - """A file containing a list of symbols to export from a shared library. + """A file containing a list of symbols to export from a shared library. The given file contains a list of symbols to be exported, and is preprocessed. @@ -1690,7 +1626,7 @@ VARIABLES = { """), 'SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of executable names. + """Compile a list of executable names. Each name in this variable corresponds to an executable built from the corresponding source file with the same base name. @@ -1701,7 +1637,7 @@ VARIABLES = { """), 'SONAME': (unicode, unicode, - """The soname of the shared object currently being linked + """The soname of the shared object currently being linked soname is the "logical name" of a shared object, often used to provide version backwards compatibility. This variable makes sense only for @@ -1709,7 +1645,7 @@ VARIABLES = { """), 'HOST_SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of host executable names. + """Compile a list of host executable names. Each name in this variable corresponds to a hosst executable built from the corresponding source file with the same base name. @@ -1720,25 +1656,20 @@ VARIABLES = { """), 'RUST_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of Rust host executable names. + """Compile a list of Rust host executable names. Each name in this variable corresponds to an executable built from the Cargo.toml in the same directory. """), 'HOST_RUST_PROGRAMS': (StrictOrderingOnAppendList, list, - """Compile a list of Rust executable names. + """Compile a list of Rust executable names. Each name in this variable corresponds to an executable built from the Cargo.toml in the same directory. """), - 'CONFIGURE_SUBST_FILES': ( - ContextDerivedTypedList( - SourcePath, - StrictOrderingOnAppendList - ), - list, + 'CONFIGURE_SUBST_FILES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, """Output files that will be generated using configure-like substitution. This is a substitute for ``AC_OUTPUT`` in autoconf. For each path in this @@ -1746,25 +1677,18 @@ VARIABLES = { ``{path}.in``. The contents of this file will be read and variable patterns like ``@foo@`` will be substituted with the values of the ``AC_SUBST`` variables declared during configure. - """ - ), + """), - 'CONFIGURE_DEFINE_FILES': ( - ContextDerivedTypedList( - SourcePath, - StrictOrderingOnAppendList - ), - list, + 'CONFIGURE_DEFINE_FILES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, """Output files generated from configure/config.status. This is a substitute for ``AC_CONFIG_HEADER`` in autoconf. This is very similar to ``CONFIGURE_SUBST_FILES`` except the generation logic takes into account the values of ``AC_DEFINE`` instead of ``AC_SUBST``. - """ - ), + """), 'EXPORTS': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be exported, and in which subdirectories. + """List of files to be exported, and in which subdirectories. ``EXPORTS`` is generally used to list the include files to be exported to ``dist/include``, but it can be used for other files as well. This variable @@ -1782,16 +1706,16 @@ VARIABLES = { ``GENERATED_FILES``. """), - 'PROGRAM': (unicode, unicode, - """Compiled executable name. + 'PROGRAM' : (unicode, unicode, + """Compiled executable name. If the configuration token ``BIN_SUFFIX`` is set, its value will be automatically appended to ``PROGRAM``. If ``PROGRAM`` already ends with ``BIN_SUFFIX``, ``PROGRAM`` will remain unchanged. """), - 'HOST_PROGRAM': (unicode, unicode, - """Compiled host executable name. + 'HOST_PROGRAM' : (unicode, unicode, + """Compiled host executable name. If the configuration token ``HOST_BIN_SUFFIX`` is set, its value will be automatically appended to ``HOST_PROGRAM``. If ``HOST_PROGRAM`` already @@ -1799,7 +1723,7 @@ VARIABLES = { """), 'DIST_INSTALL': (Enum(None, False, True), bool, - """Whether to install certain files into the dist directory. + """Whether to install certain files into the dist directory. By default, some files types are installed in the dist directory, and some aren't. Set this variable to True to force the installation of @@ -1812,7 +1736,7 @@ VARIABLES = { """), 'JAR_MANIFESTS': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """JAR manifest files that should be processed as part of the build. + """JAR manifest files that should be processed as part of the build. JAR manifests are files in the tree that define how to package files into JARs and how chrome registration is performed. For more info, @@ -1821,7 +1745,7 @@ VARIABLES = { # IDL Generation. 'XPIDL_SOURCES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """XPCOM Interface Definition Files (xpidl). + """XPCOM Interface Definition Files (xpidl). This is a list of files that define XPCOM interface definitions. Entries must be files that exist. Entries are almost certainly ``.idl`` @@ -1829,7 +1753,7 @@ VARIABLES = { """), 'XPIDL_MODULE': (unicode, unicode, - """XPCOM Interface Definition Module Name. + """XPCOM Interface Definition Module Name. This is the name of the ``.xpt`` file that is created by linking ``XPIDL_SOURCES`` together. If unspecified, it defaults to be the same @@ -1837,66 +1761,66 @@ VARIABLES = { """), 'XPCOM_MANIFESTS': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, - """XPCOM Component Manifest Files. + """XPCOM Component Manifest Files. This is a list of files that define XPCOM components to be added to the component registry. """), 'PREPROCESSED_IPDL_SOURCES': (StrictOrderingOnAppendList, list, - """Preprocessed IPDL source files. + """Preprocessed IPDL source files. These files will be preprocessed, then parsed and converted to ``.cpp`` files. """), 'IPDL_SOURCES': (StrictOrderingOnAppendList, list, - """IPDL source files. + """IPDL source files. These are ``.ipdl`` files that will be parsed and converted to ``.cpp`` files. """), 'WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """WebIDL source files. + """WebIDL source files. These will be parsed and converted to ``.cpp`` and ``.h`` files. """), 'GENERATED_EVENTS_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """WebIDL source files for generated events. + """WebIDL source files for generated events. These will be parsed and converted to ``.cpp`` and ``.h`` files. """), 'TEST_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Test WebIDL source files. + """Test WebIDL source files. These will be parsed and converted to ``.cpp`` and ``.h`` files if tests are enabled. """), 'GENERATED_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Generated WebIDL source files. + """Generated WebIDL source files. These will be generated from some other files. """), 'PREPROCESSED_TEST_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Preprocessed test WebIDL source files. + """Preprocessed test WebIDL source files. These will be preprocessed, then parsed and converted to .cpp and ``.h`` files if tests are enabled. """), 'PREPROCESSED_WEBIDL_FILES': (StrictOrderingOnAppendList, list, - """Preprocessed WebIDL source files. + """Preprocessed WebIDL source files. These will be preprocessed before being parsed and converted. """), 'WEBIDL_EXAMPLE_INTERFACES': (StrictOrderingOnAppendList, list, - """Names of example WebIDL interfaces to build as part of the build. + """Names of example WebIDL interfaces to build as part of the build. Names in this list correspond to WebIDL interface names defined in WebIDL files included in the build from one of the \*WEBIDL_FILES @@ -1905,96 +1829,96 @@ VARIABLES = { # Test declaration. 'A11Y_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining a11y tests. + """List of manifest files defining a11y tests. """), 'BROWSER_CHROME_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining browser chrome tests. + """List of manifest files defining browser chrome tests. """), 'ANDROID_INSTRUMENTATION_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining Android instrumentation tests. + """List of manifest files defining Android instrumentation tests. """), 'FIREFOX_UI_FUNCTIONAL_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining firefox-ui-functional tests. + """List of manifest files defining firefox-ui-functional tests. """), 'FIREFOX_UI_UPDATE_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining firefox-ui-update tests. + """List of manifest files defining firefox-ui-update tests. """), 'PUPPETEER_FIREFOX_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining puppeteer unit tests for Firefox. + """List of manifest files defining puppeteer unit tests for Firefox. """), 'MARIONETTE_LAYOUT_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-layout tests. + """List of manifest files defining marionette-layout tests. """), 'MARIONETTE_GPU_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-gpu tests. + """List of manifest files defining marionette-gpu tests. """), 'MARIONETTE_UNIT_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-unit tests. + """List of manifest files defining marionette-unit tests. """), 'METRO_CHROME_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining metro browser chrome tests. + """List of manifest files defining metro browser chrome tests. """), 'MOCHITEST_CHROME_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining mochitest chrome tests. + """List of manifest files defining mochitest chrome tests. """), 'MARIONETTE_DOM_MEDIA_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining marionette-media tests. + """List of manifest files defining marionette-media tests. """), 'MOCHITEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining mochitest tests. + """List of manifest files defining mochitest tests. """), 'REFTEST_MANIFESTS': (ReftestManifestList, list, - """List of manifest files defining reftests. + """List of manifest files defining reftests. These are commonly named reftest.list. """), 'CRASHTEST_MANIFESTS': (ReftestManifestList, list, - """List of manifest files defining crashtests. + """List of manifest files defining crashtests. These are commonly named crashtests.list. """), 'WEBRTC_SIGNALLING_TEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining WebRTC signalling tests. + """List of manifest files defining WebRTC signalling tests. """), 'XPCSHELL_TESTS_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining xpcshell tests. + """List of manifest files defining xpcshell tests. """), 'PYTHON_UNITTEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining python unit tests. + """List of manifest files defining python unit tests. """), 'CRAMTEST_MANIFESTS': (ManifestparserManifestList, list, - """List of manifest files defining cram unit tests. + """List of manifest files defining cram unit tests. """), # The following variables are used to control the target of installed files. 'XPI_NAME': (unicode, unicode, - """The name of an extension XPI to generate. + """The name of an extension XPI to generate. When this variable is present, the results of this directory will end up being packaged into an extension instead of the main dist/bin results. """), 'DIST_SUBDIR': (unicode, unicode, - """The name of an alternate directory to install files to. + """The name of an alternate directory to install files to. When this variable is present, the results of this directory will end up being placed in the $(DIST_SUBDIR) subdirectory of where it would @@ -2002,7 +1926,7 @@ VARIABLES = { """), 'FINAL_TARGET': (FinalTargetValue, unicode, - """The name of the directory to install targets to. + """The name of the directory to install targets to. The directory is relative to the top of the object directory. The default value is dependent on the values of XPI_NAME and DIST_SUBDIR. If @@ -2012,7 +1936,7 @@ VARIABLES = { """), 'USE_EXTENSION_MANIFEST': (bool, bool, - """Controls the name of the manifest for JAR files. + """Controls the name of the manifest for JAR files. By default, the name of the manifest is ${JAR_MANIFEST}.manifest. Setting this variable to ``True`` changes the name of the manifest to @@ -2082,7 +2006,7 @@ VARIABLES = { """), 'SPHINX_TREES': (dict, dict, - """Describes what the Sphinx documentation tree will look like. + """Describes what the Sphinx documentation tree will look like. Keys are relative directories inside the final Sphinx documentation tree to install files into. Values are directories (relative to this @@ -2090,26 +2014,26 @@ VARIABLES = { """), 'SPHINX_PYTHON_PACKAGE_DIRS': (StrictOrderingOnAppendList, list, - """Directories containing Python packages that Sphinx documents. + """Directories containing Python packages that Sphinx documents. """), 'COMPILE_FLAGS': (CompileFlags, dict, - """Recipe for compile flags for this context. Not to be manipulated + """Recipe for compile flags for this context. Not to be manipulated directly. """), 'LINK_FLAGS': (LinkFlags, dict, - """Recipe for linker flags for this context. Not to be manipulated + """Recipe for linker flags for this context. Not to be manipulated directly. """), 'ASM_FLAGS': (AsmFlags, dict, - """Recipe for linker flags for this context. Not to be manipulated + """Recipe for linker flags for this context. Not to be manipulated directly. """), 'CFLAGS': (List, list, - """Flags passed to the C compiler for all of the C source files + """Flags passed to the C compiler for all of the C source files declared in this directory. Note that the ordering of flags matters here, these flags will be @@ -2118,7 +2042,7 @@ VARIABLES = { """), 'CXXFLAGS': (List, list, - """Flags passed to the C++ compiler for all of the C++ source files + """Flags passed to the C++ compiler for all of the C++ source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2127,17 +2051,17 @@ VARIABLES = { """), 'HOST_COMPILE_FLAGS': (HostCompileFlags, dict, - """Recipe for host compile flags for this context. Not to be manipulated + """Recipe for host compile flags for this context. Not to be manipulated directly. """), 'HOST_DEFINES': (InitializedDefines, dict, - """Dictionary of compiler defines to declare for host compilation. + """Dictionary of compiler defines to declare for host compilation. See ``DEFINES`` for specifics. """), 'CMFLAGS': (List, list, - """Flags passed to the Objective-C compiler for all of the Objective-C + """Flags passed to the Objective-C compiler for all of the Objective-C source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2146,7 +2070,7 @@ VARIABLES = { """), 'CMMFLAGS': (List, list, - """Flags passed to the Objective-C++ compiler for all of the + """Flags passed to the Objective-C++ compiler for all of the Objective-C++ source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2155,7 +2079,7 @@ VARIABLES = { """), 'ASFLAGS': (List, list, - """Flags passed to the assembler for all of the assembly source files + """Flags passed to the assembler for all of the assembly source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2164,7 +2088,7 @@ VARIABLES = { """), 'HOST_CFLAGS': (List, list, - """Flags passed to the host C compiler for all of the C source files + """Flags passed to the host C compiler for all of the C source files declared in this directory. Note that the ordering of flags matters here, these flags will be @@ -2173,7 +2097,7 @@ VARIABLES = { """), 'HOST_CXXFLAGS': (List, list, - """Flags passed to the host C++ compiler for all of the C++ source files + """Flags passed to the host C++ compiler for all of the C++ source files declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2182,7 +2106,7 @@ VARIABLES = { """), 'LDFLAGS': (List, list, - """Flags passed to the linker when linking all of the libraries and + """Flags passed to the linker when linking all of the libraries and executables declared in this directory. Note that the ordering of flags matters here; these flags will be @@ -2191,7 +2115,7 @@ VARIABLES = { """), 'EXTRA_DSO_LDOPTS': (List, list, - """Flags passed to the linker when linking a shared library. + """Flags passed to the linker when linking a shared library. Note that the ordering of flags matter here, these flags will be added to the linker's command line in the same order as they @@ -2199,7 +2123,7 @@ VARIABLES = { """), 'WIN32_EXE_LDFLAGS': (List, list, - """Flags passed to the linker when linking a Windows .exe executable + """Flags passed to the linker when linking a Windows .exe executable declared in this directory. Note that the ordering of flags matter here, these flags will be @@ -2210,7 +2134,7 @@ VARIABLES = { """), 'TEST_HARNESS_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, - """List of files to be installed for test harnesses. + """List of files to be installed for test harnesses. ``TEST_HARNESS_FILES`` can be used to install files to any directory under $objdir/_tests. Files can be appended to a field to indicate @@ -2225,17 +2149,17 @@ VARIABLES = { """), 'NO_EXPAND_LIBS': (bool, bool, - """Forces to build a real static library, and no corresponding fake + """Forces to build a real static library, and no corresponding fake library. """), 'NO_COMPONENTS_MANIFEST': (bool, bool, - """Do not create a binary-component manifest entry for the + """Do not create a binary-component manifest entry for the corresponding XPCOMBinaryComponent. """), 'USE_NASM': (bool, bool, - """Use the nasm assembler to assemble assembly files from SOURCES. + """Use the nasm assembler to assemble assembly files from SOURCES. By default, the build will use the toolchain assembler, $(AS), to assemble source files in assembly language (.s or .asm files). Setting @@ -2246,7 +2170,7 @@ VARIABLES = { """), 'USE_YASM': (bool, bool, - """Use the yasm assembler to assemble assembly files from SOURCES. + """Use the yasm assembler to assemble assembly files from SOURCES. By default, the build will use the toolchain assembler, $(AS), to assemble source files in assembly language (.s or .asm files). Setting @@ -2257,7 +2181,7 @@ VARIABLES = { """), 'USE_INTEGRATED_CLANGCL_AS': (bool, bool, - """Use the integrated clang-cl assembler to assemble assembly files from SOURCES. + """Use the integrated clang-cl assembler to assemble assembly files from SOURCES. This allows using clang-cl to assemble assembly files which is useful on platforms like aarch64 where the alternative is to have to run a @@ -2269,7 +2193,7 @@ VARIABLES = { for name, (storage_type, input_types, docs) in VARIABLES.items(): if storage_type == list: raise RuntimeError('%s has a "list" storage type. Use "List" instead.' - % name) + % name) # Set of variables that are only allowed in templates: TEMPLATE_VARIABLES = { @@ -2288,7 +2212,7 @@ TEMPLATE_VARIABLES = { for name in TEMPLATE_VARIABLES: if name not in VARIABLES: raise RuntimeError('%s is in TEMPLATE_VARIABLES but not in VARIABLES.' - % name) + % name) storage_type, input_types, docs = VARIABLES[name] docs += 'This variable is only available in templates.\n' VARIABLES[name] = (storage_type, input_types, docs) @@ -2305,7 +2229,7 @@ for name in TEMPLATE_VARIABLES: # FUNCTIONS = { 'include': (lambda self: self._include, (SourcePath,), - """Include another mozbuild file in the context of this one. + """Include another mozbuild file in the context of this one. This is similar to a ``#include`` in C languages. The filename passed to the function will be read and its contents will be evaluated within the @@ -2332,7 +2256,7 @@ FUNCTIONS = { """), 'export': (lambda self: self._export, (str,), - """Make the specified variable available to all child directories. + """Make the specified variable available to all child directories. The variable specified by the argument string is added to the environment of all directories specified in the DIRS and TEST_DIRS @@ -2359,7 +2283,7 @@ FUNCTIONS = { """), 'warning': (lambda self: self._warning, (str,), - """Issue a warning. + """Issue a warning. Warnings are string messages that are printed during execution. @@ -2367,13 +2291,13 @@ FUNCTIONS = { """), 'error': (lambda self: self._error, (str,), - """Issue a fatal error. + """Issue a fatal error. If this function is called, processing is aborted immediately. """), 'template': (lambda self: self._template_decorator, (FunctionType,), - """Decorator for template declarations. + """Decorator for template declarations. Templates are a special kind of functions that can be declared in mozbuild files. Uppercase variables assigned in the function scope @@ -2432,7 +2356,7 @@ TestDirsPlaceHolder = List() # SPECIAL_VARIABLES = { 'TOPSRCDIR': (lambda context: context.config.topsrcdir, str, - """Constant defining the top source directory. + """Constant defining the top source directory. The top source directory is the parent directory containing the source code and all build files. It is typically the root directory of a @@ -2440,7 +2364,7 @@ SPECIAL_VARIABLES = { """), 'TOPOBJDIR': (lambda context: context.config.topobjdir, str, - """Constant defining the top object directory. + """Constant defining the top object directory. The top object directory is the parent directory which will contain the output of the build. This is commonly referred to as "the object @@ -2448,7 +2372,7 @@ SPECIAL_VARIABLES = { """), 'RELATIVEDIR': (lambda context: context.relsrcdir, str, - """Constant defining the relative path of this file. + """Constant defining the relative path of this file. The relative path is from ``TOPSRCDIR``. This is defined as relative to the main file being executed, regardless of whether additional @@ -2456,14 +2380,14 @@ SPECIAL_VARIABLES = { """), 'SRCDIR': (lambda context: context.srcdir, str, - """Constant defining the source directory of this file. + """Constant defining the source directory of this file. This is the path inside ``TOPSRCDIR`` where this file is located. It is the same as ``TOPSRCDIR + RELATIVEDIR``. """), 'OBJDIR': (lambda context: context.objdir, str, - """The path to the object directory for this file. + """The path to the object directory for this file. Is is the same as ``TOPOBJDIR + RELATIVEDIR``. """), @@ -2482,40 +2406,32 @@ SPECIAL_VARIABLES = { """), 'EXTRA_COMPONENTS': (lambda context: context['FINAL_TARGET_FILES'].components._strings, list, - """Additional component files to distribute. + """Additional component files to distribute. This variable contains a list of files to copy into ``$(FINAL_TARGET)/components/``. """), - 'EXTRA_PP_COMPONENTS': ( - lambda context: context['FINAL_TARGET_PP_FILES'].components._strings, - list, + 'EXTRA_PP_COMPONENTS': (lambda context: context['FINAL_TARGET_PP_FILES'].components._strings, list, """Javascript XPCOM files. This variable contains a list of files to preprocess. Generated files will be installed in the ``/components`` directory of the distribution. - """ - ), + """), - 'JS_PREFERENCE_FILES': ( - lambda context: context['FINAL_TARGET_FILES'].defaults.pref._strings, - list, + 'JS_PREFERENCE_FILES': (lambda context: context['FINAL_TARGET_FILES'].defaults.pref._strings, list, """Exported JavaScript files. A list of files copied into the dist directory for packaging and installation. Path will be defined for gre or application prefs dir based on what is building. """), - 'JS_PREFERENCE_PP_FILES': ( - lambda context: context['FINAL_TARGET_PP_FILES'].defaults.pref._strings, - list, + 'JS_PREFERENCE_PP_FILES': (lambda context: context['FINAL_TARGET_PP_FILES'].defaults.pref._strings, list, """Like JS_PREFERENCE_FILES, preprocessed.. - """ - ), + """), 'RESOURCE_FILES': (lambda context: context['FINAL_TARGET_FILES'].res, list, - """List of resources to be exported, and in which subdirectories. + """List of resources to be exported, and in which subdirectories. ``RESOURCE_FILES`` is used to list the resource files to be exported to ``dist/bin/res``, but it can be used for other files as well. This variable @@ -2529,33 +2445,30 @@ SPECIAL_VARIABLES = { RESOURCE_FILES.fonts += ['bar.res'] """), - 'CONTENT_ACCESSIBLE_FILES': ( - lambda context: context['FINAL_TARGET_FILES'].contentaccessible, - list, + 'CONTENT_ACCESSIBLE_FILES': (lambda context: context['FINAL_TARGET_FILES'].contentaccessible, list, """List of files which can be accessed by web content through resource:// URIs. ``CONTENT_ACCESSIBLE_FILES`` is used to list the files to be exported to ``dist/bin/contentaccessible``. Files can also be appended to a field to indicate which subdirectory they should be exported to. - """ - ), + """), 'EXTRA_JS_MODULES': (lambda context: context['FINAL_TARGET_FILES'].modules, list, - """Additional JavaScript files to distribute. + """Additional JavaScript files to distribute. This variable contains a list of files to copy into ``$(FINAL_TARGET)/modules. """), 'EXTRA_PP_JS_MODULES': (lambda context: context['FINAL_TARGET_PP_FILES'].modules, list, - """Additional JavaScript files to distribute. + """Additional JavaScript files to distribute. This variable contains a list of files to copy into ``$(FINAL_TARGET)/modules``, after preprocessing. """), 'TESTING_JS_MODULES': (lambda context: context['TEST_HARNESS_FILES'].modules, list, - """JavaScript modules to install in the test-only destination. + """JavaScript modules to install in the test-only destination. Some JavaScript modules (JSMs) are test-only and not distributed with Firefox. This variable defines them. @@ -2567,8 +2480,8 @@ SPECIAL_VARIABLES = { """), 'TEST_DIRS': (lambda context: context['DIRS'] if context.config.substs.get('ENABLE_TESTS') - else TestDirsPlaceHolder, list, - """Like DIRS but only for directories that contain test-only code. + else TestDirsPlaceHolder, list, + """Like DIRS but only for directories that contain test-only code. If tests are not enabled, this variable will be ignored. diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py index 0b65c0175eda..ff317b02b978 100644 --- a/python/mozbuild/mozbuild/frontend/data.py +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -15,18 +15,19 @@ contains the code for converting executed mozbuild files into these data structures. """ -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals from mozbuild.frontend.context import ( ObjDirPath, SourcePath, ) +from mozbuild.util import StrictOrderingOnAppendList from mozpack.chrome.manifest import ManifestEntry import mozpack.path as mozpath from .context import FinalTargetValue -from collections import defaultdict +from collections import defaultdict, OrderedDict import itertools from ..util import ( @@ -191,7 +192,6 @@ class ComputedFlags(ContextDerived): flags[dest_var].extend(value) return flags.items() - class XPIDLModule(ContextDerived): """Describes an XPIDL module to be compiled.""" @@ -207,7 +207,6 @@ class XPIDLModule(ContextDerived): self.name = name self.idl_files = idl_files - class BaseDefines(ContextDerived): """Context derived container object for DEFINES/HOST_DEFINES, which are OrderedDicts. @@ -233,15 +232,12 @@ class BaseDefines(ContextDerived): else: self.defines.update(more_defines) - class Defines(BaseDefines): pass - class HostDefines(BaseDefines): pass - class WebIDLCollection(ContextDerived): """Collects WebIDL info referenced during the build.""" @@ -517,8 +513,7 @@ class BaseProgram(Linkable): @property def output_path(self): if self.installed: - return ObjDirPath(self._context, '!/' + mozpath.join( - self.install_target, self.program)) + return ObjDirPath(self._context, '!/' + mozpath.join(self.install_target, self.program)) else: return ObjDirPath(self._context, '!' + self.program) @@ -681,7 +676,7 @@ class StaticLibrary(Library): ) def __init__(self, context, basename, real_name=None, - link_into=None, no_expand_lib=False): + link_into=None, no_expand_lib=False): Library.__init__(self, context, basename, real_name) self.link_into = link_into self.no_expand_lib = no_expand_lib @@ -713,8 +708,8 @@ class RustLibrary(StaticLibrary): # many other things in the build system depend on that. assert self.crate_type == 'staticlib' self.lib_name = '%s%s%s' % (context.config.rust_lib_prefix, - basename.replace('-', '_'), - context.config.rust_lib_suffix) + basename.replace('-', '_'), + context.config.rust_lib_suffix) self.dependencies = dependencies self.features = features self.target_dir = target_dir @@ -897,8 +892,8 @@ class TestManifest(ContextDerived): ) def __init__(self, context, path, manifest, flavor=None, - install_prefix=None, relpath=None, sources=(), - dupe_manifest=False): + install_prefix=None, relpath=None, sources=(), + dupe_manifest=False): ContextDerived.__init__(self, context) assert flavor in all_test_flavors() @@ -1041,13 +1036,11 @@ class UnifiedSources(BaseSources): unified_prefix = unified_prefix.replace('/', '_') suffix = self.canonical_suffix[1:] - unified_prefix = 'Unified_%s_%s' % (suffix, unified_prefix) - self.unified_source_mapping = list( - group_unified_files(source_files, - unified_prefix=unified_prefix, - unified_suffix=suffix, - files_per_unified_file=files_per_unified_file) - ) + unified_prefix='Unified_%s_%s' % (suffix, unified_prefix) + self.unified_source_mapping = list(group_unified_files(source_files, + unified_prefix=unified_prefix, + unified_suffix=suffix, + files_per_unified_file=files_per_unified_file)) class InstallationTarget(ContextDerived): @@ -1107,7 +1100,6 @@ class FinalTargetPreprocessedFiles(ContextDerived): ContextDerived.__init__(self, sandbox) self.files = files - class LocalizedFiles(FinalTargetFiles): """Sandbox container object for LOCALIZED_FILES, which is a HierarchicalStringList. @@ -1195,16 +1187,12 @@ class GeneratedFile(ContextDerived): '.inc', '.py', '.rs', - 'node.stub', # To avoid VPATH issues with installing node files: - # https://bugzilla.mozilla.org/show_bug.cgi?id=1461714#c55 - # We need to compile Java to generate JNI wrappers for native code - # compilation to consume. - 'android_apks', + 'node.stub', # To avoid VPATH issues with installing node files: https://bugzilla.mozilla.org/show_bug.cgi?id=1461714#c55 + 'android_apks', # We need to compile Java to generate JNI wrappers for native code compilation to consume. '.profdata', '.webidl' ) - self.required_for_compile = [ - f for f in self.outputs if f.endswith(suffixes) or 'stl_wrappers/' in f] + self.required_for_compile = [f for f in self.outputs if f.endswith(suffixes) or 'stl_wrappers/' in f] class ChromeManifestEntry(ContextDerived): diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index c7e7d977b382..bafe3c76f113 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import itertools import logging @@ -24,6 +24,7 @@ import pytoml from .data import ( BaseRustProgram, + BaseSources, ChromeManifestEntry, ComputedFlags, ConfigFileSubstitution, @@ -92,6 +93,7 @@ from .context import ( ObjDirPath, Path, SubContext, + TemplateContext, ) from mozbuild.base import ExecutionSummary @@ -183,8 +185,7 @@ class TreeMetadataEmitter(LoggingMixin): objs = list(emitfn(out)) self._emitter_time += time.time() - start - for o in emit_objs(objs): - yield o + for o in emit_objs(objs): yield o else: raise Exception('Unhandled output type: %s' % type(out)) @@ -195,8 +196,7 @@ class TreeMetadataEmitter(LoggingMixin): objs = list(self._emit_libs_derived(contexts)) self._emitter_time += time.time() - start - for o in emit_objs(objs): - yield o + for o in emit_objs(objs): yield o def _emit_libs_derived(self, contexts): @@ -234,11 +234,11 @@ class TreeMetadataEmitter(LoggingMixin): if isinstance(collection, WebIDLCollection): # Test webidl sources are added here as a somewhat special # case. - idl_sources[mozpath.join(root, 'test')] = [ - s for s in collection.all_test_cpp_basenames()] + idl_sources[mozpath.join(root, 'test')] = [s for s in collection.all_test_cpp_basenames()] yield collection + # Next do FINAL_LIBRARY linkage. for lib in (l for libs in self._libs.values() for l in libs): if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: @@ -255,14 +255,14 @@ class TreeMetadataEmitter(LoggingMixin): # both a static and a shared library in a directory, and having # that as a FINAL_LIBRARY. if len(set(type(l) for l in candidates)) == len(candidates) and \ - len(set(l.objdir for l in candidates)) == 1: + len(set(l.objdir for l in candidates)) == 1: for c in candidates: c.link_library(lib) else: raise SandboxValidationError( 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' 'multiple places:\n %s' % (lib.link_into, - '\n '.join(l.objdir for l in candidates)), + '\n '.join(l.objdir for l in candidates)), contexts[lib.objdir]) # ...and USE_LIBS linkage. @@ -281,13 +281,13 @@ class TreeMetadataEmitter(LoggingMixin): for lib in self._static_linking_shared: if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): shared_libs = sorted(l.basename for l in lib.linked_libraries - if isinstance(l, SharedLibrary)) + if isinstance(l, SharedLibrary)) raise SandboxValidationError( 'The static "%s" library is not used in a shared library ' 'or a program, but USE_LIBS contains the following shared ' 'library names:\n %s\n\nMaybe you can remove the ' 'static "%s" library?' % (lib.basename, - '\n '.join(shared_libs), lib.basename), + '\n '.join(shared_libs), lib.basename), contexts[lib.objdir]) # Propagate LIBRARY_DEFINES to all child libraries recursively. @@ -305,6 +305,7 @@ class TreeMetadataEmitter(LoggingMixin): propagate_defines(lib, lib.lib_defines) yield lib + for lib in (l for libs in self._libs.values() for l in libs): lib_defines = list(lib.lib_defines.get_defines()) if lib_defines: @@ -324,6 +325,7 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._binaries.values(): yield obj + LIBRARY_NAME_VAR = { 'host': 'HOST_LIBRARY_NAME', 'target': 'LIBRARY_NAME', @@ -391,14 +393,14 @@ class TreeMetadataEmitter(LoggingMixin): for d in self._external_paths: if dir.startswith('%s/' % d): candidates = [self._get_external_library(dir, name, - force_static)] + force_static)] break if not candidates: raise SandboxValidationError( '%s contains "%s", but there is no "%s" %s in %s.' % (variable, path, name, - self.LIBRARY_NAME_VAR[obj.KIND], dir), context) + self.LIBRARY_NAME_VAR[obj.KIND], dir), context) if len(candidates) > 1: # If there's more than one remaining candidate, it could be @@ -421,11 +423,11 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( '%s contains "static:%s", but there is no static ' '"%s" %s in %s.' % (variable, path, name, - self.LIBRARY_NAME_VAR[obj.KIND], dir), context) + self.LIBRARY_NAME_VAR[obj.KIND], dir), context) raise SandboxValidationError( '%s contains "static:%s", but there is no static "%s" ' '%s in the tree' % (variable, name, name, - self.LIBRARY_NAME_VAR[obj.KIND]), context) + self.LIBRARY_NAME_VAR[obj.KIND]), context) if not candidates: raise SandboxValidationError( @@ -435,23 +437,23 @@ class TreeMetadataEmitter(LoggingMixin): elif len(candidates) > 1: paths = (mozpath.join(l.relsrcdir, 'moz.build') - for l in candidates) + for l in candidates) raise SandboxValidationError( '%s contains "%s", which matches a %s defined in multiple ' 'places:\n %s' % (variable, path, - self.LIBRARY_NAME_VAR[obj.KIND], - '\n '.join(paths)), context) + self.LIBRARY_NAME_VAR[obj.KIND], + '\n '.join(paths)), context) elif force_static and not isinstance(candidates[0], StaticLibrary): raise SandboxValidationError( '%s contains "static:%s", but there is only a shared "%s" ' 'in %s. You may want to add FORCE_STATIC_LIB=True in ' - '%s/moz.build, or remove "static:".' % ( - variable, path, name, candidates[0].relobjdir, candidates[0].relobjdir), + '%s/moz.build, or remove "static:".' % (variable, path, + name, candidates[0].relobjdir, candidates[0].relobjdir), context) elif isinstance(obj, StaticLibrary) and isinstance(candidates[0], - SharedLibrary): + SharedLibrary): self._static_linking_shared.add(obj) obj.link_library(candidates[0]) @@ -483,29 +485,25 @@ class TreeMetadataEmitter(LoggingMixin): # A simple version number. if isinstance(values, (str, unicode)): raise SandboxValidationError( - '%s %s of crate %s does not list a path' % ( - description, dep_crate_name, crate_name), + '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), context) dep_path = values.get('path', None) if not dep_path: raise SandboxValidationError( - '%s %s of crate %s does not list a path' % ( - description, dep_crate_name, crate_name), + '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), context) # Try to catch the case where somebody listed a # local path for development. if os.path.isabs(dep_path): raise SandboxValidationError( - '%s %s of crate %s has a non-relative path' % ( - description, dep_crate_name, crate_name), + '%s %s of crate %s has a non-relative path' % (description, dep_crate_name, crate_name), context) if not os.path.exists(mozpath.join(context.config.topsrcdir, crate_dir, dep_path)): raise SandboxValidationError( - '%s %s of crate %s refers to a non-existent path' % ( - description, dep_crate_name, crate_name), + '%s %s of crate %s refers to a non-existent path' % (description, dep_crate_name, crate_name), context) def _rust_library(self, context, libname, static_args, cls=RustLibrary): @@ -551,6 +549,7 @@ class TreeMetadataEmitter(LoggingMixin): return cls(context, libname, cargo_file, crate_type, dependencies, features, cargo_target_dir, **static_args) + def _handle_gn_dirs(self, context): for target_dir in context.get('GN_DIRS', []): context['DIRS'] += [target_dir] @@ -570,10 +569,10 @@ class TreeMetadataEmitter(LoggingMixin): yield GnProjectData(context, target_dir, gn_dir, non_unified_sources) + def _handle_linkables(self, context, passthru, generated_files): linkables = [] host_linkables = [] - def add_program(prog, var): if var.startswith('HOST_'): host_linkables.append(prog) @@ -584,15 +583,15 @@ class TreeMetadataEmitter(LoggingMixin): if program in self._binaries: raise SandboxValidationError( 'Cannot use "%s" as %s name, ' - 'because it is already used in %s' % ( - program, kind, self._binaries[program].relsrcdir), context) + 'because it is already used in %s' % (program, kind, + self._binaries[program].relsrcdir), context) for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]: program = context.get(kind) if program: check_unique_binary(program, kind) self._binaries[program] = cls(context, program) self._linkage.append((context, self._binaries[program], - kind.replace('PROGRAM', 'USE_LIBS'))) + kind.replace('PROGRAM', 'USE_LIBS'))) add_program(self._binaries[program], kind) all_rust_programs = [] @@ -606,7 +605,7 @@ class TreeMetadataEmitter(LoggingMixin): # Verify Rust program definitions. if all_rust_programs: - config, cargo_file = self._parse_cargo_file(context) + config, cargo_file = self._parse_cargo_file(context); bin_section = config.get('bin', None) if not bin_section: raise SandboxValidationError( @@ -634,13 +633,13 @@ class TreeMetadataEmitter(LoggingMixin): if program in self._binaries: raise SandboxValidationError( 'Cannot use "%s" in %s, ' - 'because it is already used in %s' % ( - program, kind, self._binaries[program].relsrcdir), context) + 'because it is already used in %s' % (program, kind, + self._binaries[program].relsrcdir), context) self._binaries[program] = cls(context, program, - is_unit_test=kind == 'CPP_UNIT_TESTS') + is_unit_test=kind == 'CPP_UNIT_TESTS') self._linkage.append((context, self._binaries[program], - 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' - else 'USE_LIBS')) + 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' + else 'USE_LIBS')) add_program(self._binaries[program], kind) host_libname = context.get('HOST_LIBRARY_NAME') @@ -648,8 +647,8 @@ class TreeMetadataEmitter(LoggingMixin): if host_libname: if host_libname == libname: - raise SandboxValidationError( - 'LIBRARY_NAME and HOST_LIBRARY_NAME must have a different value', context) + raise SandboxValidationError('LIBRARY_NAME and ' + 'HOST_LIBRARY_NAME must have a different value', context) is_rust_library = context.get('IS_RUST_LIBRARY') if is_rust_library: @@ -780,7 +779,7 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( 'Path specified in SYMBOLS_FILE does not exist: %s ' '(resolved to %s)' % (symbols_file, - symbols_file.full_path), context) + symbols_file.full_path), context) shared_args['symbols_file'] = True else: if symbols_file.target_basename not in generated_files: @@ -804,8 +803,8 @@ class TreeMetadataEmitter(LoggingMixin): if lib.defines: defines = lib.defines.get_defines() yield GeneratedFile(context, script, - 'generate_symbols_file', lib.symbols_file, - [symbols_file], defines) + 'generate_symbols_file', lib.symbols_file, + [symbols_file], defines) if static_lib: is_rust_library = context.get('IS_RUST_LIBRARY') if is_rust_library: @@ -819,7 +818,7 @@ class TreeMetadataEmitter(LoggingMixin): if lib_defines: if not libname: raise SandboxValidationError('LIBRARY_DEFINES needs a ' - 'LIBRARY_NAME to take effect', context) + 'LIBRARY_NAME to take effect', context) lib.lib_defines.update(lib_defines) # Only emit sources if we have linkables defined in the same context. @@ -850,7 +849,7 @@ class TreeMetadataEmitter(LoggingMixin): for f in context_srcs: if f in seen_sources: raise SandboxValidationError('Source file should only ' - 'be added to %s once: %s' % (symbol, f), context) + 'be added to %s once: %s' % (symbol, f), context) seen_sources.add(f) full_path = f.full_path if isinstance(f, SourcePath): @@ -859,27 +858,26 @@ class TreeMetadataEmitter(LoggingMixin): assert isinstance(f, Path) gen_srcs.append(full_path) if symbol == 'SOURCES': - context_flags = context_srcs[f] - if context_flags: - all_flags[full_path] = context_flags + flags = context_srcs[f] + if flags: + all_flags[full_path] = flags # Files for the generation phase of PGO are unusual, so # it's not unreasonable to require them to be special. - if context_flags.pgo_generate_only: + if flags.pgo_generate_only: if not isinstance(f, Path): - raise SandboxValidationError( - 'pgo_generate_only file must not be a generated file: %s' % f, - context) + raise SandboxValidationError('pgo_generate_only file' + 'must not be a generated file: %s' % f, context) if mozpath.splitext(f)[1] != '.cpp': raise SandboxValidationError('pgo_generate_only file' - 'must be a .cpp file: %s' % f, context) - if context_flags.no_pgo: - raise SandboxValidationError( - 'pgo_generate_only files cannot be marked no_pgo: %s' % f, context) + 'must be a .cpp file: %s' % f, context) + if flags.no_pgo: + raise SandboxValidationError('pgo_generate_only files' + 'cannot be marked no_pgo: %s' % f, context) pgo_generate_only.add(f) if isinstance(f, SourcePath) and not os.path.exists(full_path): raise SandboxValidationError('File listed in %s does not ' - 'exist: \'%s\'' % (symbol, full_path), context) + 'exist: \'%s\'' % (symbol, full_path), context) # UNIFIED_SOURCES only take SourcePaths, so there should be no # generated source in here @@ -893,7 +891,7 @@ class TreeMetadataEmitter(LoggingMixin): if no_pgo: if no_pgo_sources: raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo ' - 'cannot be set at the same time', context) + 'cannot be set at the same time', context) passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo if no_pgo_sources: passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources @@ -1005,6 +1003,7 @@ class TreeMetadataEmitter(LoggingMixin): l.cxx_link = True break + def emit_from_context(self, context): """Convert a Context to tree metadata objects. @@ -1023,8 +1022,7 @@ class TreeMetadataEmitter(LoggingMixin): # We always emit a directory traversal descriptor. This is needed by # the recursive make backend. - for o in self._emit_directory_traversal_from_context(context): - yield o + for o in self._emit_directory_traversal_from_context(context): yield o for obj in self._process_xpidl(context): yield obj @@ -1054,7 +1052,7 @@ class TreeMetadataEmitter(LoggingMixin): if context.config.substs.get('OS_TARGET') == 'WINNT' and \ context['DELAYLOAD_DLLS']: context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll) - for dll in context['DELAYLOAD_DLLS']]) + for dll in context['DELAYLOAD_DLLS']]) context['OS_LIBS'].append('delayimp') for v in ['CMFLAGS', 'CMMFLAGS']: @@ -1079,7 +1077,7 @@ class TreeMetadataEmitter(LoggingMixin): raise SandboxValidationError( 'Path specified in DEFFILE does not exist: %s ' '(resolved to %s)' % (deffile, - deffile.full_path), context) + deffile.full_path), context) path = mozpath.relpath(deffile.full_path, context.objdir) else: path = deffile.target_basename @@ -1140,15 +1138,12 @@ class TreeMetadataEmitter(LoggingMixin): for path in context['CONFIGURE_SUBST_FILES']: sub = self._create_substitution(ConfigFileSubstitution, context, - path) + path) generated_files.add(str(sub.relpath)) yield sub - for defines_var, cls, backend_flags in (('DEFINES', Defines, - (computed_flags, computed_as_flags)), - ('HOST_DEFINES', HostDefines, - (computed_host_flags,)) - ): + for defines_var, cls, backend_flags in (('DEFINES', Defines, (computed_flags, computed_as_flags)), + ('HOST_DEFINES', HostDefines, (computed_host_flags,))): defines = context.get(defines_var) if defines: defines_obj = cls(context, defines) @@ -1189,18 +1184,16 @@ class TreeMetadataEmitter(LoggingMixin): full_path = local_include.full_path if not isinstance(local_include, ObjDirPath): if not os.path.exists(full_path): - raise SandboxValidationError( - 'Path specified in LOCAL_INCLUDES does not exist: %s (resolved to %s)' % - (local_include, full_path), context) + raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' + 'does not exist: %s (resolved to %s)' % (local_include, + full_path), context) if not os.path.isdir(full_path): raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' - 'is a filename, but a directory is required: %s ' - '(resolved to %s)' % (local_include, full_path), - context) + 'is a filename, but a directory is required: %s ' + '(resolved to %s)' % (local_include, full_path), context) if (full_path == context.config.topsrcdir or full_path == context.config.topobjdir): - raise SandboxValidationError( - 'Path specified in LOCAL_INCLUDES ' + raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' '(%s) resolves to the topsrcdir or topobjdir (%s), which is ' 'not allowed' % (local_include, full_path), context) include_obj = LocalInclude(context, local_include) @@ -1214,8 +1207,7 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._handle_linkables(context, passthru, generated_files): yield obj - generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) - for k in self._binaries.keys()]) + generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()]) components = [] for var, cls in ( @@ -1297,13 +1289,8 @@ class TreeMetadataEmitter(LoggingMixin): # in anything *but* LOCALIZED_FILES. if f.target_basename in localized_generated_files: raise SandboxValidationError( - ( - 'Outputs of LOCALIZED_GENERATED_FILES cannot ' - 'be used in %s: %s' - ) - % (var, f), - context, - ) + ('Outputs of LOCALIZED_GENERATED_FILES cannot be used in %s: ' + + '%s') % (var, f), context) # Addons (when XPI_NAME is defined) and Applications (when # DIST_SUBDIR is defined) use a different preferences directory @@ -1389,6 +1376,7 @@ class TreeMetadataEmitter(LoggingMixin): if context.objdir in self._host_compile_dirs: yield computed_host_flags + def _create_substitution(self, cls, context, path): sub = cls(context) sub.input_path = '%s.in' % path.full_path @@ -1407,12 +1395,12 @@ class TreeMetadataEmitter(LoggingMixin): if not xpidl_module: if context['XPIDL_SOURCES']: raise SandboxValidationError('XPIDL_MODULE must be defined if ' - 'XPIDL_SOURCES is defined.', context) + 'XPIDL_SOURCES is defined.', context) return if not context['XPIDL_SOURCES']: raise SandboxValidationError('XPIDL_MODULE cannot be defined ' - 'unless there are XPIDL_SOURCES', context) + 'unless there are XPIDL_SOURCES', context) if context['DIST_INSTALL'] is False: self.log(logging.WARN, 'mozbuild_warning', dict( @@ -1422,7 +1410,7 @@ class TreeMetadataEmitter(LoggingMixin): for idl in context['XPIDL_SOURCES']: if not os.path.exists(idl.full_path): raise SandboxValidationError('File %s from XPIDL_SOURCES ' - 'does not exist' % idl.full_path, context) + 'does not exist' % idl.full_path, context) yield XPIDLModule(context, xpidl_module, context['XPIDL_SOURCES']) @@ -1494,7 +1482,7 @@ class TreeMetadataEmitter(LoggingMixin): path = manifest_path.full_path manifest_dir = mozpath.dirname(path) manifest_reldir = mozpath.dirname(mozpath.relpath(path, - context.config.topsrcdir)) + context.config.topsrcdir)) manifest_sources = [mozpath.relpath(pth, context.config.topsrcdir) for pth in mpmanifest.source_files] install_prefix = mozpath.join(install_root, install_subdir) @@ -1502,22 +1490,22 @@ class TreeMetadataEmitter(LoggingMixin): try: if not mpmanifest.tests: raise SandboxValidationError('Empty test manifest: %s' - % path, context) + % path, context) defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] obj = TestManifest(context, path, mpmanifest, flavor=flavor, - install_prefix=install_prefix, - relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), - sources=manifest_sources, - dupe_manifest='dupe-manifest' in defaults) + install_prefix=install_prefix, + relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), + sources=manifest_sources, + dupe_manifest='dupe-manifest' in defaults) filtered = mpmanifest.tests missing = [t['name'] for t in filtered if not os.path.exists(t['path'])] if missing: raise SandboxValidationError('Test manifest (%s) lists ' - 'test that does not exist: %s' % ( - path, ', '.join(missing)), context) + 'test that does not exist: %s' % ( + path, ', '.join(missing)), context) out_dir = mozpath.join(install_prefix, manifest_reldir) if 'install-to-subdir' in defaults: @@ -1540,10 +1528,9 @@ class TreeMetadataEmitter(LoggingMixin): not os.path.isfile(mozpath.join(context.config.topsrcdir, install_path[2:])), install_path not in install_info.external_installs]): - raise SandboxValidationError( - 'Error processing test ' - 'manifest %s: entry in support-files not present ' - 'in the srcdir: %s' % (path, install_path), context) + raise SandboxValidationError('Error processing test ' + 'manifest %s: entry in support-files not present ' + 'in the srcdir: %s' % (path, install_path), context) obj.deferred_installs |= install_info.deferred_installs @@ -1554,7 +1541,7 @@ class TreeMetadataEmitter(LoggingMixin): # test package. They function as identifiers rather than files. if package_tests: manifest_relpath = mozpath.relpath(test['path'], - mozpath.dirname(test['manifest'])) + mozpath.dirname(test['manifest'])) obj.installs[mozpath.normpath(test['path'])] = \ ((mozpath.join(out_dir, manifest_relpath)), True) @@ -1580,31 +1567,29 @@ class TreeMetadataEmitter(LoggingMixin): try: del obj.installs[mozpath.join(manifest_dir, f)] except KeyError: - raise SandboxValidationError( - 'Error processing test ' + raise SandboxValidationError('Error processing test ' 'manifest %s: entry in generated-files not present ' 'elsewhere in manifest: %s' % (path, f), context) yield obj except (AssertionError, Exception): - raise SandboxValidationError( - 'Error processing test ' + raise SandboxValidationError('Error processing test ' 'manifest file %s: %s' % (path, - '\n'.join(traceback.format_exception(*sys.exc_info()))), + '\n'.join(traceback.format_exception(*sys.exc_info()))), context) def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): manifest_full_path = manifest_path.full_path manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, - context.config.topsrcdir)) + context.config.topsrcdir)) # reftest manifests don't come from manifest parser. But they are # similar enough that we can use the same emitted objects. Note # that we don't perform any installs for reftests. obj = TestManifest(context, manifest_full_path, manifest, - flavor=flavor, install_prefix='%s/' % flavor, - relpath=mozpath.join(manifest_reldir, - mozpath.basename(manifest_path))) + flavor=flavor, install_prefix='%s/' % flavor, + relpath=mozpath.join(manifest_reldir, + mozpath.basename(manifest_path))) for test, source_manifest in sorted(manifest.tests): obj.tests.append({ @@ -1623,7 +1608,7 @@ class TreeMetadataEmitter(LoggingMixin): jar_manifests = context.get('JAR_MANIFESTS', []) if len(jar_manifests) > 1: raise SandboxValidationError('While JAR_MANIFESTS is a list, ' - 'it is currently limited to one value.', context) + 'it is currently limited to one value.', context) for path in jar_manifests: yield JARManifest(context, path) @@ -1635,8 +1620,8 @@ class TreeMetadataEmitter(LoggingMixin): if os.path.exists(os.path.join(context.srcdir, 'jar.mn')): if 'jar.mn' not in jar_manifests: raise SandboxValidationError('A jar.mn exists but it ' - 'is not referenced in the moz.build file. ' - 'Please define JAR_MANIFESTS.', context) + 'is not referenced in the moz.build file. ' + 'Please define JAR_MANIFESTS.', context) def _emit_directory_traversal_from_context(self, context): o = DirectoryTraversal(context) @@ -1645,6 +1630,6 @@ class TreeMetadataEmitter(LoggingMixin): # Some paths have a subconfigure, yet also have a moz.build. Those # shouldn't end up in self._external_paths. if o.objdir: - self._external_paths -= {o.relobjdir} + self._external_paths -= { o.relobjdir } yield o diff --git a/python/mozbuild/mozbuild/frontend/gyp_reader.py b/python/mozbuild/mozbuild/frontend/gyp_reader.py index 0ec8109c9428..04efc5181108 100644 --- a/python/mozbuild/mozbuild/frontend/gyp_reader.py +++ b/python/mozbuild/mozbuild/frontend/gyp_reader.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import gyp import gyp.msvs_emulation @@ -10,6 +10,7 @@ import sys import os import time import types +import warnings import mozpack.path as mozpath from mozpack.files import FileFinder @@ -22,7 +23,10 @@ from .context import ( ) from mozbuild.util import ( expand_variables, + List, + memoize, ) +from .reader import SandboxValidationError # Define this module as gyp.generator.mozbuild so that gyp can use it # as a generator under the name "mozbuild". @@ -36,7 +40,7 @@ sys.modules['gyp.generator.mozbuild'] = sys.modules[__name__] # chrome_src for the default includes, so go backwards from the pylib # directory, which is the parent directory of gyp module. chrome_src = mozpath.abspath(mozpath.join(mozpath.dirname(gyp.__file__), - '../../../../..')) + '../../../../..')) script_dir = mozpath.join(chrome_src, 'build') @@ -70,11 +74,10 @@ class GypContext(TemplateContext): relobjdir is the object directory that will be used for this context, relative to the topobjdir defined in the ConfigEnvironment. """ - def __init__(self, config, relobjdir): self._relobjdir = relobjdir TemplateContext.__init__(self, template='Gyp', - allowed_variables=VARIABLES, config=config) + allowed_variables=VARIABLES, config=config) def handle_actions(actions, context, action_overrides): @@ -85,15 +88,10 @@ def handle_actions(actions, context, action_overrides): raise RuntimeError('GYP action %s not listed in action_overrides' % name) outputs = action['outputs'] if len(outputs) > 1: - raise NotImplementedError( - 'GYP actions with more than one output not supported: %s' % name) + raise NotImplementedError('GYP actions with more than one output not supported: %s' % name) output = outputs[0] if not output.startswith(idir): - raise NotImplementedError( - 'GYP actions outputting to somewhere other than ' - '<(INTERMEDIATE_DIR) not supported: %s' - % output - ) + raise NotImplementedError('GYP actions outputting to somewhere other than <(INTERMEDIATE_DIR) not supported: %s' % output) output = output[len(idir):] context['GENERATED_FILES'] += [output] g = context['GENERATED_FILES'][output] @@ -106,8 +104,7 @@ def handle_copies(copies, context): for copy in copies: dest = copy['destination'] if not dest.startswith(dist): - raise NotImplementedError( - 'GYP copies to somewhere other than <(PRODUCT_DIR)/dist not supported: %s' % dest) + raise NotImplementedError('GYP copies to somewhere other than <(PRODUCT_DIR)/dist not supported: %s' % dest) dest_paths = dest[len(dist):].split('/') exports = context['EXPORTS'] while dest_paths: @@ -133,8 +130,8 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, # directory. Since several targets can be in a given build_file, # separate them in subdirectories using the build_file basename # and the target_name. - reldir = mozpath.relpath(mozpath.dirname(build_file), - mozpath.dirname(path)) + reldir = mozpath.relpath(mozpath.dirname(build_file), + mozpath.dirname(path)) subdir = '%s_%s' % ( mozpath.splitext(mozpath.basename(build_file))[0], target_name, @@ -158,13 +155,12 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, target_conf = spec['configurations'][c] if 'actions' in spec: - handle_actions(spec['actions'], context, action_overrides) + handle_actions(spec['actions'], context, action_overrides) if 'copies' in spec: - handle_copies(spec['copies'], context) + handle_copies(spec['copies'], context) use_libs = [] libs = [] - def add_deps(s): for t in s.get('dependencies', []) + s.get('dependencies_original', []): ty = targets[t]['type'] @@ -175,7 +171,7 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, if ty in ('static_library', 'none'): add_deps(targets[t]) libs.extend(spec.get('libraries', [])) - # XXX: this sucks, but webrtc breaks with this right now because + #XXX: this sucks, but webrtc breaks with this right now because # it builds a library called 'gtest' and we just get lucky # that it isn't in USE_LIBS by that name anywhere. if no_chromium: @@ -183,20 +179,20 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, os_libs = [] for l in libs: - if l.startswith('-'): - os_libs.append(l) - elif l.endswith('.lib'): - os_libs.append(l[:-4]) - elif l: - # For library names passed in from moz.build. - use_libs.append(os.path.basename(l)) + if l.startswith('-'): + os_libs.append(l) + elif l.endswith('.lib'): + os_libs.append(l[:-4]) + elif l: + # For library names passed in from moz.build. + use_libs.append(os.path.basename(l)) if spec['type'] == 'none': - if not ('actions' in spec or 'copies' in spec): - continue + if not ('actions' in spec or 'copies' in spec): + continue elif spec['type'] in ('static_library', 'shared_library', 'executable'): - # Remove leading 'lib' from the target_name if any, and use as - # library name. + # Remove leading 'lib' from the target_name if any, and use as + # library name. name = spec['target_name'] if spec['type'] in ('static_library', 'shared_library'): if name.startswith('lib'): @@ -207,8 +203,7 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, context['PROGRAM'] = name.decode('utf-8') if spec['type'] == 'shared_library': context['FORCE_SHARED_LIB'] = True - elif spec['type'] == 'static_library' and \ - spec.get('variables', {}).get('no_expand_libs', '0') == '1': + elif spec['type'] == 'static_library' and spec.get('variables', {}).get('no_expand_libs', '0') == '1': # PSM links a NSS static library, but our folded libnss # doesn't actually export everything that all of the # objects within would need, so that one library @@ -227,9 +222,9 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, ext = mozpath.splitext(f)[-1] extensions.add(ext) if f.startswith('$INTERMEDIATE_DIR/'): - s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!')) + s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!')) else: - s = SourcePath(context, f) + s = SourcePath(context, f) if ext == '.h': continue if ext == '.def': @@ -256,8 +251,7 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, name, value = define.split('=', 1) # The NSS gyp file doesn't expose a way to override this # currently, so we do so here. - if name == 'NSS_ALLOW_SSLKEYLOGFILE' and \ - config.substs.get('RELEASE_OR_BETA', False): + if name == 'NSS_ALLOW_SSLKEYLOGFILE' and config.substs.get('RELEASE_OR_BETA', False): continue context['DEFINES'][name] = value else: @@ -288,8 +282,7 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, if include.startswith('/'): resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:])) elif not include.startswith(('!', '%')): - resolved = mozpath.abspath(mozpath.join( - mozpath.dirname(build_file), include)) + resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include)) if not include.startswith(('!', '%')) and not os.path.exists(resolved): continue context['LOCAL_INCLUDES'] += [include] @@ -332,17 +325,17 @@ def process_gyp_result(gyp_result, gyp_dir_attrs, path, config, output, raise NotImplementedError('Unsupported gyp target type: %s' % spec['type']) if not no_chromium: - # Add some features to all contexts. Put here in case LOCAL_INCLUDES - # order matters. - context['LOCAL_INCLUDES'] += [ - '!/ipc/ipdl/_ipdlheaders', - '/ipc/chromium/src', - '/ipc/glue', - ] - # These get set via VC project file settings for normal GYP builds. - if config.substs['OS_TARGET'] == 'WINNT': - context['DEFINES']['UNICODE'] = True - context['DEFINES']['_UNICODE'] = True + # Add some features to all contexts. Put here in case LOCAL_INCLUDES + # order matters. + context['LOCAL_INCLUDES'] += [ + '!/ipc/ipdl/_ipdlheaders', + '/ipc/chromium/src', + '/ipc/glue', + ] + # These get set via VC project file settings for normal GYP builds. + if config.substs['OS_TARGET'] == 'WINNT': + context['DEFINES']['UNICODE'] = True + context['DEFINES']['_UNICODE'] = True context['COMPILE_FLAGS']['OS_INCLUDES'] = [] for key, value in gyp_dir_attrs.sandbox_vars.items(): @@ -375,7 +368,6 @@ class GypProcessor(object): gyp dependencies will be. gyp_dir_attrs are attributes set for the dir from moz.build. """ - def __init__(self, config, gyp_dir_attrs, path, output, executor, action_overrides, non_unified_sources): self._path = path diff --git a/python/mozbuild/mozbuild/frontend/mach_commands.py b/python/mozbuild/mozbuild/frontend/mach_commands.py index e27c7aeb3ed3..0db4e7cba108 100644 --- a/python/mozbuild/mozbuild/frontend/mach_commands.py +++ b/python/mozbuild/mozbuild/frontend/mach_commands.py @@ -21,7 +21,6 @@ import mozpack.path as mozpath TOPSRCDIR = os.path.abspath(os.path.join(__file__, '../../../../../')) - class InvalidPathException(Exception): """Represents an error due to an invalid path.""" @@ -29,11 +28,11 @@ class InvalidPathException(Exception): @CommandProvider class MozbuildFileCommands(MachCommandBase): @Command('mozbuild-reference', category='build-dev', - description='View reference documentation on mozbuild files.') + description='View reference documentation on mozbuild files.') @CommandArgument('symbol', default=None, nargs='*', - help='Symbol to view help on. If not specified, all will be shown.') + help='Symbol to view help on. If not specified, all will be shown.') @CommandArgument('--name-only', '-n', default=False, action='store_true', - help='Print symbol names only.') + help='Print symbol names only.') def reference(self, symbol, name_only=False): # mozbuild.sphinx imports some Sphinx modules, so we need to be sure # the optional Sphinx package is installed. @@ -285,6 +284,7 @@ class MozbuildFileCommands(MachCommandBase): print(e.message) return 1 + def _get_files_info(self, paths, rev=None): reader = self.mozbuild_reader(config_mode='empty', vcs_revision=rev) @@ -328,6 +328,7 @@ class MozbuildFileCommands(MachCommandBase): return reader.files_info(allpaths) + @SubCommand('file-info', 'schedules', 'Show the combined SCHEDULES for the files listed.') @CommandArgument('paths', nargs='+', diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py index 30972618aca5..7eca98a0ad0d 100644 --- a/python/mozbuild/mozbuild/frontend/reader.py +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -80,6 +80,7 @@ from mozbuild.base import ExecutionSummary from concurrent.futures.process import ProcessPoolExecutor + if sys.version_info.major == 2: text_type = unicode type_type = types.TypeType @@ -105,7 +106,6 @@ class EmptyConfig(object): This variation is needed because CONFIG uses .get() to access members. Without it, None (instead of our EmptyValue types) would be returned. """ - def get(self, key, default=None): return self[key] @@ -182,7 +182,6 @@ class MozbuildSandbox(Sandbox): metadata is a dict of metadata that can be used during the sandbox evaluation. """ - def __init__(self, context, metadata={}, finder=default_finder): assert isinstance(context, Context) @@ -242,7 +241,7 @@ class MozbuildSandbox(Sandbox): # protection, so it is omitted. if not is_read_allowed(path, self._context.config): raise SandboxLoadError(self._context.source_stack, - sys.exc_info()[2], illegal_path=path) + sys.exc_info()[2], illegal_path=path) Sandbox.exec_file(self, path) @@ -291,14 +290,14 @@ class MozbuildSandbox(Sandbox): if not inspect.isfunction(func): raise Exception('`template` is a function decorator. You must ' - 'use it as `@template` preceding a function declaration.') + 'use it as `@template` preceding a function declaration.') name = func.func_name if name in self.templates: raise KeyError( 'A template named "%s" was already declared in %s.' % (name, - self.templates[name].path)) + self.templates[name].path)) if name.islower() or name.isupper() or name[0].islower(): raise NameError('Template function names must be CamelCase.') @@ -321,7 +320,6 @@ class MozbuildSandbox(Sandbox): The wrapper function does type coercion on the function arguments """ func, args_def, doc = function_def - def function(*args): def coerce(arg, type): if not isinstance(arg, type): @@ -418,7 +416,7 @@ class TemplateFunction(object): # actually never calls __getitem__ and __setitem__, so we need to # modify the AST so that accesses to globals are properly directed # to a dict. - self._global_name = b'_data' # AST wants str for this, not unicode + self._global_name = b'_data' # AST wants str for this, not unicode # In case '_data' is a name used for a variable in the function code, # prepend more underscores until we find an unused name. while (self._global_name in code.co_names or @@ -465,7 +463,6 @@ class TemplateFunction(object): """AST Node Transformer to rewrite variable accesses to go through a dict. """ - def __init__(self, sandbox, global_name): self._sandbox = sandbox self._global_name = global_name @@ -494,7 +491,6 @@ class TemplateFunction(object): class SandboxValidationError(Exception): """Represents an error encountered when validating sandbox results.""" - def __init__(self, message, context): Exception.__init__(self, message) self.context = context @@ -536,10 +532,9 @@ class BuildReaderError(Exception): MozbuildSandbox has over Sandbox (e.g. the concept of included files - which affect error messages, of course). """ - def __init__(self, file_stack, trace, sandbox_exec_error=None, - sandbox_load_error=None, validation_error=None, other_error=None, - sandbox_called_error=None): + sandbox_load_error=None, validation_error=None, other_error=None, + sandbox_called_error=None): self.file_stack = file_stack self.trace = trace @@ -564,7 +559,7 @@ class BuildReaderError(Exception): return self.file_stack[-2] if self.sandbox_error is not None and \ - len(self.sandbox_error.file_stack): + len(self.sandbox_error.file_stack): return self.sandbox_error.file_stack[-1] return self.file_stack[-1] @@ -607,7 +602,7 @@ class BuildReaderError(Exception): s.write('\n') for l in traceback.format_exception(type(self.other), self.other, - self.trace): + self.trace): s.write(unicode(l)) return s.getvalue() @@ -765,7 +760,7 @@ class BuildReaderError(Exception): if inner.args[2] in DEPRECATION_HINTS: s.write('%s\n' % - textwrap.dedent(DEPRECATION_HINTS[inner.args[2]]).strip()) + textwrap.dedent(DEPRECATION_HINTS[inner.args[2]]).strip()) return s.write('Please change the file to not use this variable.\n') @@ -807,7 +802,7 @@ class BuildReaderError(Exception): s.write(' %s\n' % inner.args[4].__name__) else: for t in inner.args[4]: - s.write(' %s\n' % t.__name__) + s.write( ' %s\n' % t.__name__) s.write('\n') s.write('Change the file to write a value of the appropriate type ') s.write('and try again.\n') @@ -1058,23 +1053,23 @@ class BuildReader(object): except SandboxCalledError as sce: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], sandbox_called_error=sce) + sys.exc_info()[2], sandbox_called_error=sce) except SandboxExecutionError as se: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], sandbox_exec_error=se) + sys.exc_info()[2], sandbox_exec_error=se) except SandboxLoadError as sle: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], sandbox_load_error=sle) + sys.exc_info()[2], sandbox_load_error=sle) except SandboxValidationError as ve: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], validation_error=ve) + sys.exc_info()[2], validation_error=ve) except Exception as e: raise BuildReaderError(list(self._execution_stack), - sys.exc_info()[2], other_error=e) + sys.exc_info()[2], other_error=e) def _read_mozbuild(self, path, config, descend, metadata): path = mozpath.normpath(path) @@ -1132,7 +1127,7 @@ class BuildReader(object): for v in ('input', 'variables'): if not getattr(gyp_dir, v): raise SandboxValidationError('Missing value for ' - 'GYP_DIRS["%s"].%s' % (target_dir, v), context) + 'GYP_DIRS["%s"].%s' % (target_dir, v), context) # The make backend assumes contexts for sub-directories are # emitted after their parent, so accumulate the gyp contexts. @@ -1145,7 +1140,7 @@ class BuildReader(object): source = SourcePath(context, s) if not self.finder.get(source.full_path): raise SandboxValidationError('Cannot find %s.' % source, - context) + context) non_unified_sources.add(source) action_overrides = {} for action, script in gyp_dir.action_overrides.iteritems(): @@ -1194,7 +1189,7 @@ class BuildReader(object): if not is_read_allowed(child_path, context.config): raise SandboxValidationError( 'Attempting to process file outside of allowed paths: %s' % - child_path, context) + child_path, context) if not descend: continue @@ -1288,7 +1283,6 @@ class BuildReader(object): # Exporting doesn't work reliably in tree traversal mode. Override # the function to no-op. functions = dict(FUNCTIONS) - def export(sandbox): return lambda varname: None functions['export'] = tuple([export] + list(FUNCTIONS['export'][1:])) @@ -1343,7 +1337,6 @@ class BuildReader(object): # times (once for every path in a directory that doesn't have any # test metadata). So, we cache the function call. defaults_cache = {} - def test_defaults_for_path(ctxs): key = tuple(ctx.current_path or ctx.main_path for ctx in ctxs) @@ -1401,8 +1394,7 @@ class BuildReader(object): test_manifest_contexts = set( ['%s_MANIFESTS' % key for key in TEST_MANIFESTS] + ['%s_MANIFESTS' % flavor.upper() for flavor in REFTEST_FLAVORS] + - ['%s_MANIFESTS' % flavor.upper().replace('-', '_') - for flavor in WEB_PLATFORM_TESTS_FLAVORS] + ['%s_MANIFESTS' % flavor.upper().replace('-', '_') for flavor in WEB_PLATFORM_TESTS_FLAVORS] ) result_context = Files(Context()) diff --git a/python/mozbuild/mozbuild/frontend/sandbox.py b/python/mozbuild/mozbuild/frontend/sandbox.py index 6e7d51890bcf..b2090802e6ea 100644 --- a/python/mozbuild/mozbuild/frontend/sandbox.py +++ b/python/mozbuild/mozbuild/frontend/sandbox.py @@ -17,7 +17,7 @@ KeyError are machine parseable. This machine-friendly data is used to present user-friendly error messages in the case of errors. """ -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import os import sys @@ -53,7 +53,6 @@ class SandboxExecutionError(SandboxError): This is a simple container exception. It's purpose is to capture state so something else can report on it. """ - def __init__(self, file_stack, exc_type, exc_value, trace): SandboxError.__init__(self, file_stack) @@ -70,7 +69,6 @@ class SandboxLoadError(SandboxError): a file. If so, the file_stack will be non-empty and the file that caused the load will be on top of the stack. """ - def __init__(self, file_stack, trace, illegal_path=None, read_error=None): SandboxError.__init__(self, file_stack) @@ -155,9 +153,9 @@ class Sandbox(dict): try: source = self._finder.get(path).read() - except Exception: + except Exception as e: raise SandboxLoadError(self._context.source_stack, - sys.exc_info()[2], read_error=path) + sys.exc_info()[2], read_error=path) self.exec_source(source, path) @@ -290,7 +288,7 @@ class Sandbox(dict): raise KeyError('global_ns', 'reassign', key) if (key not in self._context and isinstance(value, (list, dict)) - and not value): + and not value): raise KeyError('Variable %s assigned an empty value.' % key) self._context[key] = value diff --git a/python/mozbuild/mozbuild/gen_test_backend.py b/python/mozbuild/mozbuild/gen_test_backend.py index dfd3aac32736..cfb9b4715285 100644 --- a/python/mozbuild/mozbuild/gen_test_backend.py +++ b/python/mozbuild/mozbuild/gen_test_backend.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import sys from mozbuild.backend.test_manifest import TestManifestBackend diff --git a/python/mozbuild/mozbuild/generated_sources.py b/python/mozbuild/mozbuild/generated_sources.py index af8cbe496f2b..53f548ce642d 100644 --- a/python/mozbuild/mozbuild/generated_sources.py +++ b/python/mozbuild/mozbuild/generated_sources.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import hashlib import json import os diff --git a/python/mozbuild/mozbuild/gn_processor.py b/python/mozbuild/mozbuild/gn_processor.py index ce3cec4a3874..917526756c3f 100644 --- a/python/mozbuild/mozbuild/gn_processor.py +++ b/python/mozbuild/mozbuild/gn_processor.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function from collections import defaultdict from copy import deepcopy @@ -78,10 +78,7 @@ class MozbuildWriter(object): self.write('\n') self.write(self.indent + key) self.write(' += [\n ' + self.indent) - self.write( - (',\n ' + self.indent).join( - alphabetical_sorted(self.mb_serialize(v) for v in value)) - ) + self.write((',\n ' + self.indent).join(alphabetical_sorted(self.mb_serialize(v) for v in value))) self.write('\n') self.write_ln(']') @@ -115,6 +112,7 @@ class MozbuildWriter(object): if not wrote_ln: self.write_ln("%s[%s] = %s" % subst_vals) + def write_condition(self, values): def mk_condition(k, v): if not v: @@ -277,9 +275,9 @@ def process_gn_config(gn_config, srcdir, config, output, non_unified_sources, context_attrs['LOCAL_INCLUDES'] += [include] context_attrs['ASFLAGS'] = spec.get('asflags_mozilla', []) - if use_defines_in_asflags and context_attrs['DEFINES']: - context_attrs['ASFLAGS'] += ['-D' + d for d in context_attrs['DEFINES']] - flags = [_f for _f in spec.get('cflags', []) if _f in mozilla_flags] + if use_defines_in_asflags and defines: + context_attrs['ASFLAGS'] += ['-D' + d for d in defines] + flags = [f for f in spec.get('cflags', []) if f in mozilla_flags] if flags: suffix_map = { '.c': 'CFLAGS', @@ -434,6 +432,7 @@ def write_mozbuild(config, srcdir, output, non_unified_sources, gn_config_files, mb.write('\n') mb.write(generated_header) + all_attr_sets = [attrs for _, attrs in configs] all_args = [args for args, _ in configs] # Start with attributes that will be a part of the mozconfig diff --git a/python/mozbuild/mozbuild/html_build_viewer.py b/python/mozbuild/mozbuild/html_build_viewer.py index d99085842576..5151f04a4824 100644 --- a/python/mozbuild/mozbuild/html_build_viewer.py +++ b/python/mozbuild/mozbuild/html_build_viewer.py @@ -4,7 +4,7 @@ # This module contains code for running an HTTP server to view build info. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import BaseHTTPServer import json diff --git a/python/mozbuild/mozbuild/jar.py b/python/mozbuild/mozbuild/jar.py index 8a0ec99a4a71..47a2eff6390c 100644 --- a/python/mozbuild/mozbuild/jar.py +++ b/python/mozbuild/mozbuild/jar.py @@ -8,7 +8,7 @@ processing jar.mn files. See the documentation for jar.mn on MDC for further details on the format. ''' -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys import os @@ -18,6 +18,7 @@ import logging from time import localtime from MozZipFile import ZipFile from cStringIO import StringIO +from collections import defaultdict from mozbuild.preprocessor import Preprocessor from mozbuild.action.buildlist import addEntriesToListFile @@ -89,8 +90,7 @@ class JarInfo(object): self.entries = [] -class DeprecatedJarManifest(Exception): - pass +class DeprecatedJarManifest(Exception): pass class JarManifestParser(object): @@ -107,10 +107,9 @@ class JarManifestParser(object): relsrcline = re.compile('relativesrcdir\s+(?P.+?):') regline = re.compile('\%\s+(.*)$') entryre = '(?P\*)?(?P\+?)\s+' - entryline = re.compile( - entryre + ('(?P[\w\d.\-\_\\\/\+\@]+)\s*' - '(\((?P\%?)(?P[\w\d.\-\_\\\/\@\*]+)\))?\s*$') - ) + entryline = re.compile(entryre + + '(?P[\w\d.\-\_\\\/\+\@]+)\s*(\((?P\%?)(?P[\w\d.\-\_\\\/\@\*]+)\))?\s*$' + ) def __init__(self): self._current_jar = None @@ -198,7 +197,7 @@ class JarMaker(object): ''' def __init__(self, outputFormat='flat', useJarfileManifest=True, - useChromeManifest=False): + useChromeManifest=False): self.outputFormat = outputFormat self.useJarfileManifest = useJarfileManifest @@ -225,10 +224,10 @@ class JarMaker(object): p = self.pp.getCommandLineParser(unescapeDefines=True) p.add_option('-f', type='choice', default='jar', - choices=('jar', 'flat', 'symlink'), - help='fileformat used for output', - metavar='[jar, flat, symlink]', - ) + choices=('jar', 'flat', 'symlink'), + help='fileformat used for output', + metavar='[jar, flat, symlink]', + ) p.add_option('-v', action='store_true', dest='verbose', help='verbose output') p.add_option('-q', action='store_false', dest='verbose', @@ -239,14 +238,14 @@ class JarMaker(object): p.add_option('-s', type='string', action='append', default=[], help='source directory') p.add_option('-t', type='string', help='top source directory') - p.add_option('-c', '--l10n-src', type='string', - action='append', help='localization directory') + p.add_option('-c', '--l10n-src', type='string', action='append' + , help='localization directory') p.add_option('--l10n-base', type='string', action='store', help='base directory to be used for localization (requires relativesrcdir)' ) - p.add_option('--locale-mergedir', type='string', action='store', - help='base directory to be used for l10n-merge ' - '(requires l10n-base and relativesrcdir)' + p.add_option('--locale-mergedir', type='string', action='store' + , + help='base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)' ) p.add_option('--relativesrcdir', type='string', help='relativesrcdir to be used for localization') @@ -293,18 +292,18 @@ class JarMaker(object): chromeDir = \ os.path.basename(os.path.dirname(os.path.normpath(chromeManifest))) logging.info("adding '%s' entry to root chrome manifest appid=%s" - % (chromeDir, self.rootManifestAppId)) + % (chromeDir, self.rootManifestAppId)) addEntriesToListFile(rootChromeManifest, ['manifest %s/chrome.manifest application=%s' % (chromeDir, - self.rootManifestAppId)]) + self.rootManifestAppId)]) def updateManifest(self, manifestPath, chromebasepath, register): '''updateManifest replaces the % in the chrome registration entries with the given chrome base path, and updates the given manifest file. ''' myregister = dict.fromkeys(map(lambda s: s.replace('%', - chromebasepath), register)) + chromebasepath), register)) addEntriesToListFile(manifestPath, myregister.iterkeys()) def makeJar(self, infile, jardir): @@ -315,7 +314,7 @@ class JarMaker(object): ''' # making paths absolute, guess srcdir if file and add to sourcedirs - def _normpath(p): return os.path.normpath(os.path.abspath(p)) + _normpath = lambda p: os.path.normpath(os.path.abspath(p)) self.topsourcedir = _normpath(self.topsourcedir) self.sourcedirs = [_normpath(p) for p in self.sourcedirs] if self.localedirs: @@ -349,7 +348,7 @@ class JarMaker(object): if self.l10nmerge or not self.l10nbase: # add en-US if we merge, or if it's not l10n locdirs.append(os.path.join(self.topsourcedir, - relativesrcdir, 'en-US')) + relativesrcdir, 'en-US')) return locdirs def processJarSection(self, jarinfo, jardir): @@ -458,7 +457,7 @@ class JarMaker(object): if jf is not None: jf.close() raise RuntimeError('File "{0}" not found in {1}'.format(src, - ', '.join(src_base))) + ', '.join(src_base))) if out in self._seen_output: raise RuntimeError('%s already added' % out) @@ -502,7 +501,7 @@ class JarMaker(object): try: info = self.jarfile.getinfo(aPath) return info.date_time - except Exception: + except: return 0 def getOutput(self, name): @@ -586,7 +585,7 @@ def main(args=None): jm.l10nmerge = options.locale_mergedir if jm.l10nmerge and not os.path.isdir(jm.l10nmerge): logging.warning("WARNING: --locale-mergedir passed, but '%s' does not exist. " - "Ignore this message if the locale is complete." % jm.l10nmerge) + "Ignore this message if the locale is complete." % jm.l10nmerge) elif options.locale_mergedir: p.error('l10n-base required when using locale-mergedir') jm.localedirs = options.l10n_src diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index c733fc19e8f0..221d957867ae 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -108,12 +108,12 @@ class Watch(MachCommandBase): if not conditions.is_artifact_build(self): print('mach watch requires an artifact build. See ' - 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build') # noqa + 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Simple_Firefox_build') return 1 if not self.substs.get('WATCHMAN', None): print('mach watch requires watchman to be installed. See ' - 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') # noqa + 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') return 1 self._activate_virtualenv() @@ -121,7 +121,7 @@ class Watch(MachCommandBase): self.virtualenv_manager.install_pip_package('pywatchman==1.3.0') except Exception: print('Could not install pywatchman from pip. See ' - 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') # noqa + 'https://developer.mozilla.org/docs/Mozilla/Developer_guide/Build_Instructions/Incremental_builds_with_filesystem_watching') return 1 from mozbuild.faster_daemon import Daemon @@ -140,19 +140,19 @@ class Build(MachCommandBase): @Command('build', category='build', description='Build the tree.') @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, - help='Number of concurrent jobs to run. Default is the number of CPUs.') + help='Number of concurrent jobs to run. Default is the number of CPUs.') @CommandArgument('-C', '--directory', default=None, - help='Change to a subdirectory of the build directory first.') + help='Change to a subdirectory of the build directory first.') @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP) @CommandArgument('-X', '--disable-extra-make-dependencies', default=False, action='store_true', help='Do not add extra make dependencies.') @CommandArgument('-v', '--verbose', action='store_true', - help='Verbose output for what commands the build is running.') + help='Verbose output for what commands the build is running.') @CommandArgument('--keep-going', action='store_true', help='Keep building after an error has occurred') def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, - directory=None, verbose=False, keep_going=False): + directory=None, verbose=False, keep_going=False): """Build the source tree. With no arguments, this will perform a full build. @@ -190,7 +190,7 @@ class Build(MachCommandBase): mach_context=self._mach_context) @Command('configure', category='build', - description='Configure the tree (run configure and config.status).') + description='Configure the tree (run configure and config.status).') @CommandArgument('options', default=None, nargs=argparse.REMAINDER, help='Configure options') def configure(self, options=None, buildstatus_messages=False, line_handler=None): @@ -207,15 +207,15 @@ class Build(MachCommandBase): line_handler=line_handler) @Command('resource-usage', category='post-build', - description='Show information about system resource usage for a build.') + description='Show information about system resource usage for a build.') @CommandArgument('--address', default='localhost', - help='Address the HTTP server should listen on.') + help='Address the HTTP server should listen on.') @CommandArgument('--port', type=int, default=0, - help='Port number the HTTP server should listen on.') + help='Port number the HTTP server should listen on.') @CommandArgument('--browser', default='firefox', - help='Web browser to automatically open. See webbrowser Python module.') + help='Web browser to automatically open. See webbrowser Python module.') @CommandArgument('--url', - help='URL of JSON document to display') + help='URL of JSON document to display') def resource_usage(self, address=None, port=None, browser=None, url=None): import webbrowser from mozbuild.html_build_viewer import BuildViewerServer @@ -228,8 +228,8 @@ class Build(MachCommandBase): last = self._get_state_filename('build_resources.json') if not os.path.exists(last): print('Build resources not available. If you have performed a ' - 'build and receive this message, the psutil Python package ' - 'likely failed to initialize properly.') + 'build and receive this message, the psutil Python package ' + 'likely failed to initialize properly.') return 1 server.add_resource_json_file('last', last) @@ -246,17 +246,17 @@ class Build(MachCommandBase): server.run() @Command('build-backend', category='build', - description='Generate a backend used to build the tree.') + description='Generate a backend used to build the tree.') @CommandArgument('-d', '--diff', action='store_true', - help='Show a diff of changes.') + help='Show a diff of changes.') # It would be nice to filter the choices below based on # conditions, but that is for another day. @CommandArgument('-b', '--backend', nargs='+', choices=sorted(backends), - help='Which backend to build.') + help='Which backend to build.') @CommandArgument('-v', '--verbose', action='store_true', - help='Verbose output.') + help='Verbose output.') @CommandArgument('-n', '--dry-run', action='store_true', - help='Do everything except writing files out.') + help='Do everything except writing files out.') def build_backend(self, backend, diff=False, verbose=False, dry_run=False): python = self.virtualenv_manager.python_path config_status = os.path.join(self.topobjdir, 'config.status') @@ -279,7 +279,7 @@ class Build(MachCommandBase): args.append('--dry-run') return self._run_command_in_objdir(args=args, pass_thru=True, - ensure_exit_code=False) + ensure_exit_code=False) @CommandProvider @@ -295,7 +295,7 @@ class CargoProvider(MachCommandBase): @SubCommand('cargo', 'check', description='Run `cargo check` on a given crate. Defaults to gkrust.') @CommandArgument('--all-crates', default=None, action='store_true', - help='Check all of the crates in the tree.') + help='Check all of the crates in the tree.') @CommandArgument('crates', default=None, nargs='*', help='The crate name(s) to check.') def check(self, all_crates=None, crates=None): # XXX duplication with `mach vendor rust` @@ -310,7 +310,7 @@ class CargoProvider(MachCommandBase): if all_crates: crates = crates_and_roots.keys() - elif crates is None or crates == []: + elif crates == None or crates == []: crates = ['gkrust'] for crate in crates: @@ -335,33 +335,30 @@ class CargoProvider(MachCommandBase): return 0 - @CommandProvider class Doctor(MachCommandBase): """Provide commands for diagnosing common build environment problems""" @Command('doctor', category='devenv', - description='') + description='') @CommandArgument('--fix', default=None, action='store_true', - help='Attempt to fix found problems.') + help='Attempt to fix found problems.') def doctor(self, fix=None): self._activate_virtualenv() from mozbuild.doctor import Doctor doctor = Doctor(self.topsrcdir, self.topobjdir, fix) return doctor.check_all() - @CommandProvider class Clobber(MachCommandBase): NO_AUTO_LOG = True CLOBBER_CHOICES = ['objdir', 'python'] - @Command('clobber', category='build', - description='Clobber the tree (delete the object directory).') + description='Clobber the tree (delete the object directory).') @CommandArgument('what', default=['objdir'], nargs='*', - help='Target to clobber, must be one of {{{}}} (default objdir).'.format( + help='Target to clobber, must be one of {{{}}} (default objdir).'.format( ', '.join(CLOBBER_CHOICES))) @CommandArgument('--full', action='store_true', - help='Perform a full clobber') + help='Perform a full clobber') def clobber(self, what, full=False): """Clean up the source and object directories. @@ -395,10 +392,10 @@ class Clobber(MachCommandBase): Clobberer(self.topsrcdir, self.topobjdir).remove_objdir(full) except OSError as e: if sys.platform.startswith('win'): - if isinstance(e, WindowsError) and e.winerror in (5, 32): + if isinstance(e, WindowsError) and e.winerror in (5,32): self.log(logging.ERROR, 'file_access_error', {'error': e}, - "Could not clobber because a file was in use. If the " - "application is running, try closing it. {error}") + "Could not clobber because a file was in use. If the " + "application is running, try closing it. {error}") return 1 raise @@ -424,17 +421,16 @@ class Clobber(MachCommandBase): except BuildEnvironmentNotFoundException: return {} - @CommandProvider class Logs(MachCommandBase): """Provide commands to read mach logs.""" NO_AUTO_LOG = True @Command('show-log', category='post-build', - description='Display mach logs') + description='Display mach logs') @CommandArgument('log_file', nargs='?', type=argparse.FileType('rb'), - help='Filename to read log data from. Defaults to the log of the last ' - 'mach command.') + help='Filename to read log data from. Defaults to the log of the last ' + 'mach command.') def show_log(self, log_file=None): if not log_file: path = self._get_state_filename('last_log.json') @@ -506,12 +502,12 @@ class Warnings(MachCommandBase): return database @Command('warnings-summary', category='post-build', - description='Show a summary of compiler warnings.') + description='Show a summary of compiler warnings.') @CommandArgument('-C', '--directory', default=None, - help='Change to a subdirectory of the build directory first.') + help='Change to a subdirectory of the build directory first.') @CommandArgument('report', default=None, nargs='?', - help='Warnings report to display. If not defined, show the most ' - 'recent report.') + help='Warnings report to display. If not defined, show the most ' + 'recent report.') def summary(self, directory=None, report=None): database = self.database @@ -524,7 +520,7 @@ class Warnings(MachCommandBase): type_counts = database.type_counts(dirpath) sorted_counts = sorted(type_counts.iteritems(), - key=operator.itemgetter(1)) + key=operator.itemgetter(1)) total = 0 for k, v in sorted_counts: @@ -534,14 +530,14 @@ class Warnings(MachCommandBase): print('%d\tTotal' % total) @Command('warnings-list', category='post-build', - description='Show a list of compiler warnings.') + description='Show a list of compiler warnings.') @CommandArgument('-C', '--directory', default=None, - help='Change to a subdirectory of the build directory first.') + help='Change to a subdirectory of the build directory first.') @CommandArgument('--flags', default=None, nargs='+', - help='Which warnings flags to match.') + help='Which warnings flags to match.') @CommandArgument('report', default=None, nargs='?', - help='Warnings report to display. If not defined, show the most ' - 'recent report.') + help='Warnings report to display. If not defined, show the most ' + 'recent report.') def list(self, directory=None, flags=None, report=None): database = self.database @@ -572,12 +568,11 @@ class Warnings(MachCommandBase): continue if warning['column'] is not None: - print('%s:%d:%d [%s] %s' % ( - filename, warning['line'], warning['column'], - warning['flag'], warning['message'])) + print('%s:%d:%d [%s] %s' % (filename, warning['line'], + warning['column'], warning['flag'], warning['message'])) else: print('%s:%d [%s] %s' % (filename, warning['line'], - warning['flag'], warning['message'])) + warning['flag'], warning['message'])) def join_ensure_dir(self, dir1, dir2): dir1 = mozpath.normpath(dir1) @@ -588,49 +583,48 @@ class Warnings(MachCommandBase): print('Specified directory not found.') return None - @CommandProvider class GTestCommands(MachCommandBase): @Command('gtest', category='testing', - description='Run GTest unit tests (C++ tests).') + description='Run GTest unit tests (C++ tests).') @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', - help="test_filter is a ':'-separated list of wildcard patterns " - "(called the positive patterns), optionally followed by a '-' " - "and another ':'-separated pattern list (called the negative patterns).") + help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," + "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") @CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int, - help='Run the tests in parallel using multiple processes.') + help='Run the tests in parallel using multiple processes.') @CommandArgument('--tbpl-parser', '-t', action='store_true', - help='Output test results in a format that can be parsed by TBPL.') + help='Output test results in a format that can be parsed by TBPL.') @CommandArgument('--shuffle', '-s', action='store_true', - help='Randomize the execution order of tests.') + help='Randomize the execution order of tests.') + @CommandArgument('--package', - default='org.mozilla.geckoview.test', - help='(Android only) Package name of test app.') + default='org.mozilla.geckoview.test', + help='(Android only) Package name of test app.') @CommandArgument('--adbpath', - dest='adb_path', - help='(Android only) Path to adb binary.') + dest='adb_path', + help='(Android only) Path to adb binary.') @CommandArgument('--deviceSerial', - dest='device_serial', - help="(Android only) adb serial number of remote device. " - "Required when more than one device is connected to the host. " - "Use 'adb devices' to see connected devices.") + dest='device_serial', + help="(Android only) adb serial number of remote device. " + "Required when more than one device is connected to the host. " + "Use 'adb devices' to see connected devices.") @CommandArgument('--remoteTestRoot', - dest='remote_test_root', - help='(Android only) Remote directory to use as test root ' - '(eg. /mnt/sdcard/tests or /data/local/tests).') + dest='remote_test_root', + help='(Android only) Remote directory to use as test root ' + '(eg. /mnt/sdcard/tests or /data/local/tests).') @CommandArgument('--libxul', - dest='libxul_path', - help='(Android only) Path to gtest libxul.so.') + dest='libxul_path', + help='(Android only) Path to gtest libxul.so.') + @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', - help='Enable the debugger. Not specifying a --debugger option will result in ' - 'the default debugger being used.') + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', - help='Name of debugger to use.') + help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, - group='debugging', - help='Command-line arguments to pass to the debugger itself; ' - 'split as the Bourne shell would.') + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') + def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, package, adb_path, device_serial, remote_test_root, libxul_path, debug, debugger, debugger_args): @@ -669,17 +663,16 @@ class GTestCommands(MachCommandBase): if debug or debugger or debugger_args: print("--debug options are not supported on Android and will be ignored") return self.android_gtest(cwd, shuffle, gtest_filter, - package, adb_path, device_serial, - remote_test_root, libxul_path) + package, adb_path, device_serial, remote_test_root, libxul_path) if package or adb_path or device_serial or remote_test_root or libxul_path: print("One or more Android-only options will be ignored") app_path = self.get_binary_path('app') - args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe'] + args = [app_path, '-unittest', '--gtest_death_test_style=threadsafe']; if sys.platform.startswith('win') and \ - 'MOZ_LAUNCHER_PROCESS' in self.defines: + 'MOZ_LAUNCHER_PROCESS' in self.defines: args.append('--wait-for-browser') if debug or debugger or debugger_args: @@ -717,7 +710,6 @@ class GTestCommands(MachCommandBase): from mozprocess import ProcessHandlerMixin import functools - def handle_line(job_id, line): # Prepend the jobId line = '[%d] %s' % (job_id + 1, line.strip()) @@ -728,11 +720,10 @@ class GTestCommands(MachCommandBase): for i in range(0, jobs): gtest_env["GTEST_SHARD_INDEX"] = str(i) processes[i] = ProcessHandlerMixin([app_path, "-unittest"], - cwd=cwd, - env=gtest_env, - processOutputLine=[ - functools.partial(handle_line, i)], - universal_newlines=True) + cwd=cwd, + env=gtest_env, + processOutputLine=[functools.partial(handle_line, i)], + universal_newlines=True) processes[i].run() exit_code = 0 @@ -754,7 +745,7 @@ class GTestCommands(MachCommandBase): from mozlog.commandline import setup_logging format_args = {'level': self._mach_context.settings['test']['level']} default_format = self._mach_context.settings['test']['format'] - setup_logging('mach-gtest', {}, {default_format: sys.stdout}, format_args) + log = setup_logging('mach-gtest', {}, {default_format: sys.stdout}, format_args) # ensure that a device is available and test app is installed from mozrunner.devices.android_device import (verify_android_device, get_adb_path) @@ -817,11 +808,10 @@ class GTestCommands(MachCommandBase): args = [debuggerInfo.path] + debuggerInfo.args + args return args - @CommandProvider class ClangCommands(MachCommandBase): @Command('clang-complete', category='devenv', - description='Generate a .clang_complete file.') + description='Generate a .clang_complete file.') def clang_complete(self): import shlex @@ -876,9 +866,9 @@ class Package(MachCommandBase): """Package the built product for distribution.""" @Command('package', category='post-build', - description='Package the built product for distribution as an APK, DMG, etc.') + description='Package the built product for distribution as an APK, DMG, etc.') @CommandArgument('-v', '--verbose', action='store_true', - help='Verbose output for what commands the packaging process is running.') + help='Verbose output for what commands the packaging process is running.') def package(self, verbose=False): ret = self._run_make(directory=".", target='package', silent=not verbose, ensure_exit_code=False) @@ -886,15 +876,14 @@ class Package(MachCommandBase): self.notify('Packaging complete') return ret - @CommandProvider class Install(MachCommandBase): """Install a package.""" @Command('install', category='post-build', - description='Install the package on the machine, or on a device.') + description='Install the package on the machine, or on a device.') @CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output when installing to an Android emulator.') + help='Print verbose output when installing to an Android emulator.') def install(self, verbose=False): if conditions.is_android(self): from mozrunner.devices.android_device import verify_android_device @@ -904,7 +893,6 @@ class Install(MachCommandBase): self.notify('Install complete') return ret - @SettingsProvider class RunSettings(): config_settings = [ @@ -915,7 +903,6 @@ single quoted to force them to be strings. """.strip()), ] - @CommandProvider class RunProgram(MachCommandBase): """Run the compiled program.""" @@ -923,68 +910,58 @@ class RunProgram(MachCommandBase): prog_group = 'the compiled program' @Command('run', category='post-build', - description='Run the compiled program, possibly under a debugger or DMD.') + description='Run the compiled program, possibly under a debugger or DMD.') @CommandArgument('params', nargs='...', group=prog_group, - help='Command-line arguments to be passed through to the program. Not ' - 'specifying a --profile or -P option will result in a temporary profile ' - 'being used.') + help='Command-line arguments to be passed through to the program. Not specifying a --profile or -P option will result in a temporary profile being used.') @CommandArgumentGroup(prog_group) @CommandArgument('--remote', '-r', action='store_true', group=prog_group, - help='Do not pass the --no-remote argument by default.') + help='Do not pass the --no-remote argument by default.') @CommandArgument('--background', '-b', action='store_true', group=prog_group, - help='Do not pass the --foreground argument by default on Mac.') + help='Do not pass the --foreground argument by default on Mac.') @CommandArgument('--noprofile', '-n', action='store_true', group=prog_group, - help='Do not pass the --profile argument by default.') + help='Do not pass the --profile argument by default.') @CommandArgument('--disable-e10s', action='store_true', group=prog_group, - help='Run the program with electrolysis disabled.') + help='Run the program with electrolysis disabled.') @CommandArgument('--enable-crash-reporter', action='store_true', group=prog_group, - help='Run the program with the crash reporter enabled.') + help='Run the program with the crash reporter enabled.') @CommandArgument('--setpref', action='append', default=[], group=prog_group, - help='Set the specified pref before starting the program. Can be set ' - 'multiple times. Prefs can also be set in ~/.mozbuild/machrc in the ' - '[runprefs] section - see `./mach settings` for more information.') + help='Set the specified pref before starting the program. Can be set multiple times. Prefs can also be set in ~/.mozbuild/machrc in the [runprefs] section - see `./mach settings` for more information.') @CommandArgument('--temp-profile', action='store_true', group=prog_group, - help='Run the program using a new temporary profile created inside ' - 'the objdir.') + help='Run the program using a new temporary profile created inside the objdir.') @CommandArgument('--macos-open', action='store_true', group=prog_group, - help="On macOS, run the program using the open(1) command. Per open(1), " - "the browser is launched \"just as if you had double-clicked the file's " - "icon\". The browser can not be launched under a debugger with this option.") + help="On macOS, run the program using the open(1) command. Per open(1), the browser is launched \"just as if you had double-clicked the file's icon\". The browser can not be launched under a debugger with this option.") + @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', - help='Enable the debugger. Not specifying a --debugger option will result ' - 'in the default debugger being used.') + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', - help='Name of debugger to use.') + help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, - group='debugging', - help='Command-line arguments to pass to the debugger itself; ' - 'split as the Bourne shell would.') + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') @CommandArgument('--debugparams', action=StoreDebugParamsAndWarnAction, - default=None, type=str, dest='debugger_args', group='debugging', - help=argparse.SUPPRESS) + default=None, type=str, dest='debugger_args', group='debugging', + help=argparse.SUPPRESS) + @CommandArgumentGroup('DMD') @CommandArgument('--dmd', action='store_true', group='DMD', - help='Enable DMD. The following arguments have no effect without this.') + help='Enable DMD. The following arguments have no effect without this.') @CommandArgument('--mode', choices=['live', 'dark-matter', 'cumulative', 'scan'], group='DMD', - help='Profiling mode. The default is \'dark-matter\'.') + help='Profiling mode. The default is \'dark-matter\'.') @CommandArgument('--stacks', choices=['partial', 'full'], group='DMD', - help='Allocation stack trace coverage. The default is \'partial\'.') + help='Allocation stack trace coverage. The default is \'partial\'.') @CommandArgument('--show-dump-stats', action='store_true', group='DMD', - help='Show stats when doing dumps.') + help='Show stats when doing dumps.') def run(self, params, remote, background, noprofile, disable_e10s, - enable_crash_reporter, setpref, temp_profile, macos_open, debug, - debugger, debugger_args, dmd, mode, stacks, show_dump_stats): + enable_crash_reporter, setpref, temp_profile, macos_open, debug, + debugger, debugger_args, dmd, mode, stacks, show_dump_stats): if conditions.is_android(self): # Running Firefox for Android is completely different if dmd: print("DMD is not supported for Firefox for Android") return 1 - from mozrunner.devices.android_device import ( - verify_android_device, - run_firefox_for_android - ) + from mozrunner.devices.android_device import verify_android_device, run_firefox_for_android if not (debug or debugger or debugger_args): verify_android_device(self, install=True) return run_firefox_for_android(self, params) @@ -998,7 +975,7 @@ class RunProgram(MachCommandBase): binpath = self.get_binary_path('app') except Exception as e: print("It looks like your program isn't built.", - "You can run |mach build| to build it.") + "You can run |mach build| to build it.") print(e) return 1 @@ -1006,7 +983,7 @@ class RunProgram(MachCommandBase): if macos_open: if debug: print("The browser can not be launched in the debugger " - "when using the macOS open command.") + "when using the macOS open command.") return 1 try: m = re.search(r'^.+\.app', binpath) @@ -1014,7 +991,7 @@ class RunProgram(MachCommandBase): args = ['open', apppath, '--args'] except Exception as e: print("Couldn't get the .app path from the binary path. " - "The macOS open option can only be used on macOS") + "The macOS open option can only be used on macOS") print(e) return 1 else: @@ -1133,23 +1110,21 @@ class RunProgram(MachCommandBase): extra_env['DMD'] = '1' return self.run_process(args=args, ensure_exit_code=False, - pass_thru=True, append_env=extra_env) - + pass_thru=True, append_env=extra_env) @CommandProvider class Buildsymbols(MachCommandBase): """Produce a package of debug symbols suitable for use with Breakpad.""" @Command('buildsymbols', category='post-build', - description='Produce a package of Breakpad-format symbols.') + description='Produce a package of Breakpad-format symbols.') def buildsymbols(self): return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False) - @CommandProvider class Makefiles(MachCommandBase): @Command('empty-makefiles', category='build-dev', - description='Find empty Makefile.in in the tree.') + description='Find empty Makefile.in in the tree.') def empty(self): import pymake.parser import pymake.parserdata @@ -1189,7 +1164,7 @@ class Makefiles(MachCommandBase): relpath = os.path.relpath(path, self.topsrcdir) try: statements = [s for s in pymake.parser.parsefile(path) - if is_statement_relevant(s)] + if is_statement_relevant(s)] if not statements: print(relpath) @@ -1202,18 +1177,17 @@ class Makefiles(MachCommandBase): if f == 'Makefile.in': yield os.path.join(root, f) - @CommandProvider class MachDebug(MachCommandBase): @Command('environment', category='build-dev', - description='Show info about the mach and build environment.') + description='Show info about the mach and build environment.') @CommandArgument('--format', default='pretty', - choices=['pretty', 'configure', 'json'], - help='Print data in the given format.') + choices=['pretty', 'configure', 'json'], + help='Print data in the given format.') @CommandArgument('--output', '-o', type=str, - help='Output to the given file.') + help='Output to the given file.') @CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output.') + help='Print verbose output.') def environment(self, format, output=None, verbose=False): func = getattr(self, '_environment_%s' % format.replace('.', '_')) @@ -1278,7 +1252,6 @@ class MachDebug(MachCommandBase): def _environment_json(self, out, verbose): import json - class EnvironmentEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, MozbuildObject): @@ -1302,11 +1275,11 @@ class ArtifactSubCommand(SubCommand): after = SubCommand.__call__(self, func) args = [ CommandArgument('--tree', metavar='TREE', type=str, - help='Firefox tree.'), + help='Firefox tree.'), CommandArgument('--job', metavar='JOB', choices=JOB_CHOICES, - help='Build job.'), + help='Build job.'), CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output.'), + help='Print verbose output.'), ] for arg in args: after = arg(after) @@ -1326,7 +1299,7 @@ class PackageFrontend(MachCommandBase): """Fetch and install binary artifacts from Mozilla automation.""" @Command('artifact', category='post-build', - description='Use pre-built artifacts to build Firefox.') + description='Use pre-built artifacts to build Firefox.') def artifact(self): '''Download, cache, and install pre-built binary artifacts to build Firefox. @@ -1369,15 +1342,15 @@ class PackageFrontend(MachCommandBase): return artifacts @ArtifactSubCommand('artifact', 'install', - 'Install a good pre-built artifact.') + 'Install a good pre-built artifact.') @CommandArgument('source', metavar='SRC', nargs='?', type=str, - help='Where to fetch and install artifacts from. Can be omitted, in ' - 'which case the current hg repository is inspected; an hg revision; ' - 'a remote URL; or a local file.', - default=None) + help='Where to fetch and install artifacts from. Can be omitted, in ' + 'which case the current hg repository is inspected; an hg revision; ' + 'a remote URL; or a local file.', + default=None) @CommandArgument('--skip-cache', action='store_true', - help='Skip all local caches to force re-fetching remote artifacts.', - default=False) + help='Skip all local caches to force re-fetching remote artifacts.', + default=False) @CommandArgument('--no-tests', action='store_true', help="Don't install tests.") @CommandArgument('--symbols', nargs='?', action=SymbolsAction, help='Download symbols.') @CommandArgument('--host-bins', action='store_true', help='Download host binaries.') @@ -1393,7 +1366,7 @@ class PackageFrontend(MachCommandBase): return artifacts.install_from(source, distdir or self.distdir) @ArtifactSubCommand('artifact', 'clear-cache', - 'Delete local artifacts and reset local artifact cache.') + 'Delete local artifacts and reset local artifact cache.') def artifact_clear_cache(self, tree=None, job=None, verbose=False): self._set_log_level(verbose) artifacts = self._make_artifacts(tree=tree, job=job) @@ -1402,35 +1375,35 @@ class PackageFrontend(MachCommandBase): @SubCommand('artifact', 'toolchain') @CommandArgument('--verbose', '-v', action='store_true', - help='Print verbose output.') + help='Print verbose output.') @CommandArgument('--cache-dir', metavar='DIR', - help='Directory where to store the artifacts cache') + help='Directory where to store the artifacts cache') @CommandArgument('--skip-cache', action='store_true', - help='Skip all local caches to force re-fetching remote artifacts.', - default=False) + help='Skip all local caches to force re-fetching remote artifacts.', + default=False) @CommandArgument('--from-build', metavar='BUILD', nargs='+', - help='Download toolchains resulting from the given build(s); ' - 'BUILD is a name of a toolchain task, e.g. linux64-clang') + help='Download toolchains resulting from the given build(s); ' + 'BUILD is a name of a toolchain task, e.g. linux64-clang') @CommandArgument('--tooltool-manifest', metavar='MANIFEST', - help='Explicit tooltool manifest to process') + help='Explicit tooltool manifest to process') @CommandArgument('--authentication-file', metavar='FILE', - help='Use the RelengAPI token found in the given file to authenticate') + help='Use the RelengAPI token found in the given file to authenticate') @CommandArgument('--tooltool-url', metavar='URL', - help='Use the given url as tooltool server') + help='Use the given url as tooltool server') @CommandArgument('--no-unpack', action='store_true', - help='Do not unpack any downloaded file') + help='Do not unpack any downloaded file') @CommandArgument('--retry', type=int, default=4, - help='Number of times to retry failed downloads') + help='Number of times to retry failed downloads') @CommandArgument('--artifact-manifest', metavar='FILE', - help='Store a manifest about the downloaded taskcluster artifacts') + help='Store a manifest about the downloaded taskcluster artifacts') @CommandArgument('files', nargs='*', - help='A list of files to download, in the form path@task-id, in ' - 'addition to the files listed in the tooltool manifest.') + help='A list of files to download, in the form path@task-id, in ' + 'addition to the files listed in the tooltool manifest.') def artifact_toolchain(self, verbose=False, cache_dir=None, - skip_cache=False, from_build=(), - tooltool_manifest=None, authentication_file=None, - tooltool_url=None, no_unpack=False, retry=None, - artifact_manifest=None, files=()): + skip_cache=False, from_build=(), + tooltool_manifest=None, authentication_file=None, + tooltool_url=None, no_unpack=False, retry=None, + artifact_manifest=None, files=()): '''Download, cache and install pre-built toolchains. ''' from mozbuild.artifacts import ArtifactCache @@ -1518,7 +1491,7 @@ class PackageFrontend(MachCommandBase): name = os.path.basename(artifact_name) artifact_url = get_artifact_url(task_id, artifact_name, - use_proxy=not artifact_name.startswith('public/')) + use_proxy=not artifact_name.startswith('public/')) super(ArtifactRecord, self).__init__( artifact_url, name, None, digest, algorithm, unpack=True) @@ -1694,7 +1667,6 @@ class PackageFrontend(MachCommandBase): return 0 - class StaticAnalysisSubCommand(SubCommand): def __call__(self, func): after = SubCommand.__call__(self, func) @@ -1754,7 +1726,7 @@ class StaticAnalysisMonitor(object): try: warning = self._warnings_collector.process_line(line) - except Exception: + except: pass if line.find('clang-tidy') != -1: @@ -1835,8 +1807,8 @@ class StaticAnalysis(MachCommandBase): help='Output format to write in a file') @CommandArgument('--outgoing', default=False, action='store_true', help='Run static analysis checks on outgoing files from mercurial repository') - def check(self, source=None, jobs=2, strip=1, verbose=False, checks='-*', - fix=False, header_filter='', output=None, format='text', outgoing=False): + def check(self, source=None, jobs=2, strip=1, verbose=False, + checks='-*', fix=False, header_filter='', output=None, format='text', outgoing=False): from mozbuild.controller.building import ( StaticAnalysisFooter, StaticAnalysisOutputManager, @@ -1850,9 +1822,8 @@ class StaticAnalysis(MachCommandBase): return rc if self._is_version_eligible() is False: - self.log(logging.ERROR, 'static-analysis', {}, - "You're using an old version of clang-format binary." - " Please update to a more recent one by running: './mach bootstrap'") + self.log(logging.ERROR, 'static-analysis', {}, "You're using an old version of clang-format binary." + " Please update to a more recent one by running: './mach bootstrap'") return 1 rc = self._build_compile_db(verbose=verbose) @@ -1880,8 +1851,8 @@ class StaticAnalysis(MachCommandBase): if not total: self.log(logging.INFO, 'static-analysis', {}, - "There are no files eligible for analysis. Please note that 'header' files " - "cannot be used for analysis since they do not consist compilation units.") + "There are no files eligible for analysis. Please note that 'header' files " + "cannot be used for analysis since they do not consist compilation units.") return 0 cwd = self.topobjdir @@ -1891,13 +1862,11 @@ class StaticAnalysis(MachCommandBase): args = self._get_clang_tidy_command( checks=checks, header_filter=header_filter, sources=source, jobs=jobs, fix=fix) - monitor = StaticAnalysisMonitor( - self.topsrcdir, self.topobjdir, self._clang_tidy_config, total) + monitor = StaticAnalysisMonitor(self.topsrcdir, self.topobjdir, self._clang_tidy_config, total) footer = StaticAnalysisFooter(self.log_manager.terminal, monitor) with StaticAnalysisOutputManager(self.log_manager, monitor, footer) as output_manager: - rc = self.run_process(args=args, ensure_exit_code=False, - line_handler=output_manager.on_line, cwd=cwd) + rc = self.run_process(args=args, ensure_exit_code=False, line_handler=output_manager.on_line, cwd=cwd) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_db)}, @@ -1926,18 +1895,16 @@ class StaticAnalysis(MachCommandBase): help='Write coverity output translated to json output in a file') @CommandArgument('--coverity_output_path', '-co', default=None, help='Path where to write coverity results as cov-results.json. ' - 'If no path is specified the default path from the coverity working ' - 'directory, ~./mozbuild/coverity is used.') + 'If no path is specified the default path from the coverity working directory, ' + '~./mozbuild/coverity is used.') @CommandArgument('--outgoing', default=False, action='store_true', help='Run coverity on outgoing files from mercurial or git repository') - def check_coverity(self, source=[], output=None, coverity_output_path=None, - outgoing=False, verbose=False): + def check_coverity(self, source=[], output=None, coverity_output_path=None, outgoing=False, verbose=False): self._set_log_level(verbose) self.log_manager.enable_all_structured_loggers() if 'MOZ_AUTOMATION' not in os.environ: - self.log(logging.INFO, 'static-analysis', {}, - 'Coverity based static-analysis cannot be ran outside automation.') + self.log(logging.INFO, 'static-analysis', {}, 'Coverity based static-analysis cannot be ran outside automation.') return # Use outgoing files instead of source files @@ -1947,8 +1914,7 @@ class StaticAnalysis(MachCommandBase): source = map(os.path.abspath, files) if len(source) == 0: - self.log(logging.ERROR, 'static-analysis', {}, - 'There are no files that coverity can use to scan.') + self.log(logging.ERROR, 'static-analysis', {}, 'There are no files that coverity can use to scan.') return 0 rc = self._build_compile_db(verbose=verbose) @@ -1959,8 +1925,7 @@ class StaticAnalysis(MachCommandBase): commands_list = self.get_files_with_commands(source) if len(commands_list) == 0: - self.log(logging.INFO, 'static-analysis', {}, - 'There are no files that need to be analyzed.') + self.log(logging.INFO, 'static-analysis', {}, 'There are no files that need to be analyzed.') return 0 # Load the configuration file for coverity static-analysis @@ -1974,37 +1939,31 @@ class StaticAnalysis(MachCommandBase): # First run cov-run-desktop --setup in order to setup the analysis env cmd = [self.cov_run_desktop, '--setup'] - self.log(logging.INFO, 'static-analysis', {}, - 'Running {} --setup'.format(self.cov_run_desktop)) + self.log(logging.INFO, 'static-analysis', {}, 'Running {} --setup'.format(self.cov_run_desktop)) rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True) if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, - 'Running {} --setup failed!'.format(self.cov_run_desktop)) + self.log(logging.ERROR, 'static-analysis', {}, 'Running {} --setup failed!'.format(self.cov_run_desktop)) return rc # Run cov-configure for clang cmd = [self.cov_configure, '--clang'] - self.log(logging.INFO, 'static-analysis', {}, - 'Running {} --clang'.format(self.cov_configure)) + self.log(logging.INFO, 'static-analysis', {}, 'Running {} --clang'.format(self.cov_configure)) rc = self.run_process(args=cmd, cwd=self.cov_path, pass_thru=True) if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, - 'Running {} --clang failed!'.format(self.cov_configure)) + self.log(logging.ERROR, 'static-analysis', {}, 'Running {} --clang failed!'.format(self.cov_configure)) return rc # For each element in commands_list run `cov-translate` for element in commands_list: cmd = [self.cov_translate, '--dir', self.cov_idir_path] + element['command'].split(' ') - self.log(logging.INFO, 'static-analysis', {}, - 'Running Coverity Tranlate for {}'.format(cmd)) + self.log(logging.INFO, 'static-analysis', {}, 'Running Coverity Tranlate for {}'.format(cmd)) rc = self.run_process(args=cmd, cwd=element['directory'], pass_thru=True) if rc != 0: - self.log(logging.ERROR, 'static-analysis', {}, - 'Running Coverity Tranlate failed for {}'.format(cmd)) + self.log(logging.ERROR, 'static-analysis', {}, 'Running Coverity Tranlate failed for {}'.format(cmd)) return cmd if coverity_output_path is None: @@ -2014,8 +1973,7 @@ class StaticAnalysis(MachCommandBase): # Once the capture is performed we need to do the actual Coverity Desktop analysis cmd = [self.cov_run_desktop, '--json-output-v6', cov_result, '--analyze-captured-source'] - self.log(logging.INFO, 'static-analysis', {}, - 'Running Coverity Analysis for {}'.format(cmd)) + self.log(logging.INFO, 'static-analysis', {}, 'Running Coverity Analysis for {}'.format(cmd)) rc = self.run_process(cmd, cwd=self.cov_state_path, pass_thru=True) if rc != 0: self.log(logging.ERROR, 'static-analysis', {}, 'Coverity Analysis failed!') @@ -2026,25 +1984,22 @@ class StaticAnalysis(MachCommandBase): def get_reliability_index_for_cov_checker(self, checker_name): if self._cov_config is None: self.log(logging.INFO, 'static-analysis', {}, 'Coverity config file not found, ' - 'using default-value \'reliablity\' = medium. for checker {}'.format( - checker_name)) + 'using default-value \'reliablity\' = medium. for checker {}'.format(checker_name)) return 'medium' checkers = self._cov_config['coverity_checkers'] if checker_name not in checkers: self.log(logging.INFO, 'static-analysis', {}, - 'Coverity checker {} not found to determine reliability index. ' - 'For the moment we shall use the default \'reliablity\' = medium.'.format( - checker_name)) + 'Coverity checker {} not found to determine reliability index. ' + 'For the moment we shall use the default \'reliablity\' = medium.'.format(checker_name)) return 'medium' if 'reliability' not in checkers[checker_name]: # This checker doesn't have a reliability index self.log(logging.INFO, 'static-analysis', {}, - 'Coverity checker {} doesn\'t have a reliability index set, ' - 'field \'reliability is missing\', please cosinder adding it. ' - 'For the moment we shall use the default \'reliablity\' = medium.'.format( - checker_name)) + 'Coverity checker {} doesn\'t have a reliability index set, ' + 'field \'reliability is missing\', please cosinder adding it. ' + 'For the moment we shall use the default \'reliablity\' = medium.'.format(checker_name)) return 'medium' return checkers[checker_name]['reliability'] @@ -2061,16 +2016,13 @@ class StaticAnalysis(MachCommandBase): def build_element(issue): # We look only for main event - event_path = next( - (event for event in issue['events'] if event['main'] is True), None) + event_path = next((event for event in issue['events'] if event['main'] is True), None) dict_issue = { 'line': issue['mainEventLineNumber'], 'flag': issue['checkerName'], 'message': event_path['eventDescription'], - 'reliability': self.get_reliability_index_for_cov_checker( - issue['checkerName'] - ), + 'reliability': self.get_reliability_index_for_cov_checker(issue['checkerName']), 'extra': { 'category': issue['checkerProperties']['category'], 'stateOnServer': issue['stateOnServer'], @@ -2080,11 +2032,10 @@ class StaticAnalysis(MachCommandBase): # Embed all events into extra message for event in issue['events']: - dict_issue['extra']['stack'].append( - {'file_path': event['strippedFilePathname'], - 'line_number': event['lineNumber'], - 'path_type': event['eventTag'], - 'description': event['eventDescription']}) + dict_issue['extra']['stack'].append({'file_path': event['strippedFilePathname'], + 'line_number': event['lineNumber'], + 'path_type': event['eventTag'], + 'description': event['eventDescription']}) return dict_issue @@ -2092,12 +2043,8 @@ class StaticAnalysis(MachCommandBase): path = self.cov_is_file_in_source(issue['strippedMainEventFilePathname'], source) if path is None: # Since we skip a result we should log it - self.log(logging.INFO, 'static-analysis', {}, - 'Skipping CID: {0} from file: {1} since it\'s not related ' - 'with the current patch.'.format( - issue['stateOnServer']['cid'], - issue['strippedMainEventFilePathname']) - ) + self.log(logging.INFO, 'static-analysis', {}, 'Skipping CID: {0} from file: {1} since it\'s not related with the current patch.'.format( + issue['stateOnServer']['cid'], issue['strippedMainEventFilePathname'])) continue if path in files_list: files_list[path]['warnings'].append(build_element(issue)) @@ -2113,8 +2060,7 @@ class StaticAnalysis(MachCommandBase): secret_name = 'project/relman/coverity' secrets_url = '{}/secrets/v1/secret/{}'.format(get_root_url(True), secret_name) - self.log(logging.INFO, 'static-analysis', {}, - 'Using symbol upload token from the secrets service: "{}"'.format(secrets_url)) + self.log(logging.INFO, 'static-analysis', {}, 'Using symbol upload token from the secrets service: "{}"'.format(secrets_url)) import requests res = requests.get(secrets_url) @@ -2123,8 +2069,7 @@ class StaticAnalysis(MachCommandBase): cov_config = secret['secret'] if 'secret' in secret else None if cov_config is None: - self.log(logging.ERROR, 'static-analysis', {}, - 'Ill formatted secret for Coverity. Aborting analysis.') + self.log(logging.ERROR, 'static-analysis', {}, 'Ill formatted secret for Coverity. Aborting analysis.') return 1 self.cov_analysis_url = cov_config.get('package_url') @@ -2211,8 +2156,7 @@ class StaticAnalysis(MachCommandBase): self.cov_idir_path = mozpath.join(self.cov_work_path, self.cov_package_ver, 'idir') if not os.path.exists(self.cov_path): - self.log(logging.ERROR, 'static-analysis', {}, - 'Missing Coverity in {}'.format(self.cov_path)) + self.log(logging.ERROR, 'static-analysis', {}, 'Missing Coverity in {}'.format(self.cov_path)) return 1 return 0 @@ -2382,7 +2326,7 @@ class StaticAnalysis(MachCommandBase): if item['publish']: checkers.append(item['name']) tp_path = mozpath.join(self.topsrcdir, config['third_party']) - except Exception: + except Exception as e: print('Looks like config.yaml is not valid, so we are unable ' 'to determine default checkers, and which folder to ' 'exclude, using defaults provided by infer') @@ -2407,8 +2351,8 @@ class StaticAnalysis(MachCommandBase): config = yaml.safe_load(file_handler) except Exception: self.log(logging.ERROR, 'static-analysis', {}, - 'Looks like config.yaml is not valid, we are going to use default' - ' values for the rest of the analysis for clang-tidy.') + 'Looks like config.yaml is not valid, we are going to use default' + ' values for the rest of the analysis for clang-tidy.') return None return config @@ -2418,8 +2362,8 @@ class StaticAnalysis(MachCommandBase): config = yaml.safe_load(file_handler) except Exception: self.log(logging.ERROR, 'static-analysis', {}, - 'Looks like config.yaml is not valid, we are going to use default' - ' values for the rest of the analysis for coverity.') + 'Looks like config.yaml is not valid, we are going to use default' + ' values for the rest of the analysis for coverity.') return None return config @@ -2432,8 +2376,7 @@ class StaticAnalysis(MachCommandBase): if 'package_version' in self._clang_tidy_config: version = self._clang_tidy_config['package_version'] else: - self.log(logging.ERROR, 'static-analysis', {}, - "Unable to find 'package_version' in the config.yml") + self.log(logging.ERROR, 'static-analysis', {}, "Unable to find 'package_version' in the config.yml") return False # Because the fact that we ship together clang-tidy and clang-format @@ -2448,8 +2391,7 @@ class StaticAnalysis(MachCommandBase): return True except subprocess.CalledProcessError as e: self.log(logging.ERROR, 'static-analysis', {}, - "Error determining the version clang-tidy/format binary, please see the " - "attached exception: \n{}".format(e.output)) + "Error determining the version clang-tidy/format binary, please see the attached exception: \n{}".format(e.output)) return False @@ -2468,7 +2410,7 @@ class StaticAnalysis(MachCommandBase): # is considered to be the source in order to limit the diagnostic message to # the source files or folders. common_args += ['-header-filter=%s' % (header_filter - if len(header_filter) else '|'.join(sources))] + if len(header_filter) else '|'.join(sources))] # From our configuration file, config.yaml, we build the configuration list, for # the checkers that are used. These configuration options are used to better fit @@ -2593,11 +2535,8 @@ class StaticAnalysis(MachCommandBase): platform, _ = self.platform if platform not in self._clang_tidy_config['platforms']: - self.log( - logging.ERROR, 'static-analysis', {}, - "RUNNING: clang-tidy autotest for platform {} not supported.".format( - platform) - ) + self.log(logging.ERROR, 'static-analysis', {}, + "RUNNING: clang-tidy autotest for platform {} not supported.".format(platform)) return self.TOOLS_UNSUPORTED_PLATFORM import concurrent.futures @@ -2627,14 +2566,12 @@ class StaticAnalysis(MachCommandBase): # 1. Checker attribute 'publish' is False. not_published = not bool(item.get('publish', True)) # 2. Checker has restricted-platforms and current platform is not of them. - ignored_platform = ('restricted-platforms' in item and - platform not in item['restricted-platforms']) + ignored_platform = 'restricted-platforms' in item and platform not in item['restricted-platforms'] # 3. Checker name is mozilla-* or -*. ignored_checker = item['name'] in ['mozilla-*', '-*'] # 4. List checker_names is passed and the current checker is not part of the # list or 'publish' is False - checker_not_in_list = checker_names and ( - item['name'] not in checker_names or not_published) + checker_not_in_list = checker_names and (item['name'] not in checker_names or not_published) if not_published or \ ignored_platform or \ ignored_checker or \ @@ -2655,8 +2592,7 @@ class StaticAnalysis(MachCommandBase): if error_code != self.TOOLS_SUCCESS: - self.log(logging.INFO, 'static-analysis', {}, - "FAIL: the following clang-tidy check(s) failed:") + self.log(logging.INFO, 'static-analysis', {}, "FAIL: the following clang-tidy check(s) failed:") for failure in checkers_results: checker_error = failure['checker-error'] checker_name = failure['checker-name'] @@ -2666,29 +2602,18 @@ class StaticAnalysis(MachCommandBase): message_to_log = '' if checker_error == self.TOOLS_CHECKER_NOT_FOUND: - message_to_log = \ - "\tChecker {} not present in this clang-tidy version.".format( - checker_name) + message_to_log = "\tChecker {} not present in this clang-tidy version.".format( + checker_name) elif checker_error == self.TOOLS_CHECKER_NO_TEST_FILE: - message_to_log = \ - "\tChecker {0} does not have a test file - {0}.cpp".format( - checker_name) + message_to_log = "\tChecker {0} does not have a test file - {0}.cpp".format(checker_name) elif checker_error == self.TOOLS_CHECKER_RETURNED_NO_ISSUES: - message_to_log = ( - "\tChecker {0} did not find any issues in its test file, " - "clang-tidy output for the run is:\n{1}" - ).format(checker_name, info1) + message_to_log = "\tChecker {0} did not find any issues in its test file, clang-tidy output for the run is:\n{1}".format( + checker_name, info1) elif checker_error == self.TOOLS_CHECKER_RESULT_FILE_NOT_FOUND: - message_to_log = \ - "\tChecker {0} does not have a result file - {0}.json".format( - checker_name) + message_to_log = "\tChecker {0} does not have a result file - {0}.json".format(checker_name) elif checker_error == self.TOOLS_CHECKER_DIFF_FAILED: - message_to_log = ( - "\tChecker {0}\nExpected: {1}\n" - "Got: {2}\n" - "clang-tidy output for the run is:\n" - "{3}" - ).format(checker_name, info1, info2, info3) + message_to_log = "\tChecker {0}\nExpected: {1}\nGot: {2}\nclang-tidy output for the run is:\n{3}".format( + checker_name, info1, info2, info3) print('\n'+message_to_log) @@ -2722,19 +2647,14 @@ class StaticAnalysis(MachCommandBase): return self._parse_issues(clang_output), clang_output def _run_analysis_batch(self, items): - self.log(logging.INFO, 'static-analysis', {}, - "RUNNING: clang-tidy checker batch analysis.") + self.log(logging.INFO, 'static-analysis', {},"RUNNING: clang-tidy checker batch analysis.") if not len(items): - self.log(logging.ERROR, 'static-analysis', {}, - "ERROR: clang-tidy checker list is empty!") + self.log(logging.ERROR, 'static-analysis', {}, "ERROR: clang-tidy checker list is empty!") return self.TOOLS_CHECKER_LIST_EMPTY issues, clang_output = self._run_analysis( - checks='-*,' + ",".join(items), - header_filter='', - sources=[mozpath.join(self._clang_tidy_base_path, "test", checker) + '.cpp' - for checker in items], - print_out=True) + checks='-*,' + ",".join(items), header_filter='', + sources=[mozpath.join(self._clang_tidy_base_path, "test", checker) + '.cpp' for checker in items], print_out=True) if issues is None: return self.TOOLS_CHECKER_FAILED_FILE @@ -2742,8 +2662,7 @@ class StaticAnalysis(MachCommandBase): failed_checks = [] failed_checks_baseline = [] for checker in items: - test_file_path_json = mozpath.join( - self._clang_tidy_base_path, "test", checker) + '.json' + test_file_path_json = mozpath.join(self._clang_tidy_base_path, "test", checker) + '.json' # Read the pre-determined issues baseline_issues = self._get_autotest_stored_issues(test_file_path_json) @@ -2757,15 +2676,12 @@ class StaticAnalysis(MachCommandBase): failed_checks_baseline.append(baseline_issues) if len(failed_checks) > 0: - self.log(logging.ERROR, 'static-analysis', {}, - 'The following check(s) failed for bulk analysis: ' + ' '.join(failed_checks)) + self.log(logging.ERROR, 'static-analysis', {}, 'The following check(s) failed for bulk analysis: ' + ' '.join(failed_checks)) for failed_check, baseline_issue in zip(failed_checks, failed_checks_baseline): - print('\tChecker {0} expect following results: \n\t\t{1}'.format( - failed_check, baseline_issue)) + print('\tChecker {0} expect following results: \n\t\t{1}'.format(failed_check, baseline_issue)) - print('This is the output generated by clang-tidy for the bulk build:\n{}'.format( - clang_output)) + print('This is the output generated by clang-tidy for the bulk build:\n{}'.format(clang_output)) return self.TOOLS_CHECKER_DIFF_FAILED return self.TOOLS_SUCCESS @@ -2926,11 +2842,9 @@ class StaticAnalysis(MachCommandBase): @CommandArgument('--force', action='store_true', help='Force re-install even though the tool exists in mozbuild.', default=False) - @CommandArgument('--minimal-install', action='store_true', - help='Download only clang based tool.', + @CommandArgument('--minimal-install', action='store_true', help='Download only clang based tool.', default=False) - def install(self, source=None, skip_cache=False, force=False, minimal_install=False, - verbose=False): + def install(self, source=None, skip_cache=False, force=False, minimal_install=False, verbose=False): self._set_log_level(verbose) rc = self._get_clang_tools(force=force, skip_cache=skip_cache, source=source, verbose=verbose) @@ -2994,14 +2908,12 @@ class StaticAnalysis(MachCommandBase): @CommandArgument('--output', '-o', default=None, dest='output_path', help='Specify a file handle to write clang-format raw output instead of ' 'applying changes. This can be stdout or a file path.') - @CommandArgument('--format', '-f', choices=('diff', 'json'), default='diff', - dest='output_format', + @CommandArgument('--format', '-f', choices=('diff', 'json'), default='diff', dest='output_format', help='Specify the output format used: diff is the raw patch provided by ' 'clang-format, json is a list of atomic changes to process.') @CommandArgument('--outgoing', default=False, action='store_true', help='Run clang-format on outgoing files from mercurial repository') - def clang_format(self, assume_filename, path, commit, output_path=None, output_format='diff', - verbose=False, outgoing=False): + def clang_format(self, assume_filename, path, commit, output_path=None, output_format='diff', verbose=False, outgoing=False): # Run clang-format or clang-format-diff on the local changes # or files/directories if path is None and outgoing: @@ -3010,7 +2922,7 @@ class StaticAnalysis(MachCommandBase): if path: # Create the full path list - def path_maker(f_name): return os.path.join(self.topsrcdir, f_name) + path_maker = lambda f_name: os.path.join(self.topsrcdir, f_name) path = map(path_maker, path) os.chdir(self.topsrcdir) @@ -3042,9 +2954,8 @@ class StaticAnalysis(MachCommandBase): return rc if self._is_version_eligible() is False: - self.log(logging.ERROR, 'static-analysis', {}, - "You're using an old version of clang-format binary." - " Please update to a more recent one by running: './mach bootstrap'") + self.log(logging.ERROR, 'static-analysis', {}, "You're using an old version of clang-format binary." + " Please update to a more recent one by running: './mach bootstrap'") return 1 if path is None: @@ -3052,8 +2963,7 @@ class StaticAnalysis(MachCommandBase): self._clang_format_path, commit, output) if assume_filename: - return self._run_clang_format_in_console(self._clang_format_path, - path, assume_filename) + return self._run_clang_format_in_console(self._clang_format_path, path, assume_filename) return self._run_clang_format_path(self._clang_format_path, path, output, output_format) @@ -3076,7 +2986,7 @@ class StaticAnalysis(MachCommandBase): } # Verify if this checker actually exists - if check not in self._clang_tidy_checks: + if not check in self._clang_tidy_checks: checker_error['checker-error'] = self.TOOLS_CHECKER_NOT_FOUND checkers_results.append(checker_error) return self.TOOLS_CHECKER_NOT_FOUND @@ -3176,7 +3086,7 @@ class StaticAnalysis(MachCommandBase): checks += ',' + item['name'] except Exception: print('Looks like config.yaml is not valid, so we are unable to ' - 'determine default checkers, using \'-checks=-*,mozilla-*\'') + 'determine default checkers, using \'-checks=-*,mozilla-*\'') checks += ',mozilla-*' finally: return checks @@ -3200,7 +3110,7 @@ class StaticAnalysis(MachCommandBase): checker_config['CheckOptions'] = config_list except Exception: print('Looks like config.yaml is not valid, so we are unable to ' - 'determine configuration for checkers, so using default') + 'determine configuration for checkers, so using default') checker_config = None finally: return checker_config @@ -3220,7 +3130,7 @@ class StaticAnalysis(MachCommandBase): ran_configure = True try: config = self.config_environment - except Exception: + except Exception as e: pass return (0, config, ran_configure) @@ -3282,10 +3192,10 @@ class StaticAnalysis(MachCommandBase): self._clang_apply_replacements = mozpath.join( self._clang_tools_path, "clang-tidy", "bin", "clang-apply-replacements" + config.substs.get('BIN_SUFFIX', '')) - self._run_clang_tidy_path = mozpath.join(self._clang_tools_path, "clang-tidy", - "share", "clang", "run-clang-tidy.py") - self._clang_format_diff = mozpath.join(self._clang_tools_path, "clang-tidy", - "share", "clang", "clang-format-diff.py") + self._run_clang_tidy_path = mozpath.join(self._clang_tools_path, "clang-tidy", "share", "clang", + "run-clang-tidy.py") + self._clang_format_diff = mozpath.join(self._clang_tools_path, "clang-tidy", "share", "clang", + "clang-format-diff.py") return 0 def _do_clang_tools_exist(self): @@ -3311,8 +3221,8 @@ class StaticAnalysis(MachCommandBase): # and start from scratch. shutil.rmtree(self._clang_tools_path) return self._get_clang_tools(force=force, skip_cache=skip_cache, - source=source, verbose=verbose, - download_if_needed=download_if_needed) + source=source, verbose=verbose, + download_if_needed=download_if_needed) # Create base directory where we store clang binary os.mkdir(self._clang_tools_path) @@ -3338,10 +3248,10 @@ class StaticAnalysis(MachCommandBase): currentWorkingDir = os.getcwd() os.chdir(self._clang_tools_path) rc = self._artifact_manager.artifact_toolchain(verbose=verbose, - skip_cache=skip_cache, - from_build=[job], - no_unpack=False, - retry=0) + skip_cache=skip_cache, + from_build=[job], + no_unpack=False, + retry=0) # Change back the cwd os.chdir(currentWorkingDir) @@ -3383,7 +3293,7 @@ class StaticAnalysis(MachCommandBase): args += ['--include', 'glob:**{0}'.format(dot_extension)] args += ['--exclude', 'listfile:{0}'.format(self._format_ignore_file)] else: - commit_range = "HEAD" # All uncommitted changes. + commit_range = "HEAD" # All uncommitted changes. if commit: commit_range = commit if ".." in commit else "{}~..{}".format(commit, commit) args = ["git", "diff", "--no-color", "-U0", commit_range, "--"] @@ -3596,8 +3506,7 @@ class StaticAnalysis(MachCommandBase): # here, we expect changes. if we are here, this means that # there is a diff to show if e.output: - # Replace the temp path by the path relative to the repository to - # display a valid patch + # Replace the temp path by the path relative to the repository to display a valid patch relative_path = os.path.relpath(original_path, self.topsrcdir) patch = e.output.replace(target_file, relative_path) patch = patch.replace(original_path, relative_path) @@ -3699,41 +3608,38 @@ class Vendor(MachCommandBase): @SubCommand('vendor', 'rust', description='Vendor rust crates from crates.io into third_party/rust') @CommandArgument('--ignore-modified', action='store_true', - help='Ignore modified files in current checkout', - default=False) + help='Ignore modified files in current checkout', + default=False) @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true', - help='Permit overly-large files to be added to the repository', - default=False) + help='Permit overly-large files to be added to the repository', + default=False) def vendor_rust(self, **kwargs): from mozbuild.vendor_rust import VendorRust vendor_command = self._spawn(VendorRust) vendor_command.vendor(**kwargs) @SubCommand('vendor', 'aom', - description='Vendor av1 video codec reference implementation into the ' - 'source repository.') + description='Vendor av1 video codec reference implementation into the source repository.') @CommandArgument('-r', '--revision', - help='Repository tag or commit to update to.') + help='Repository tag or commit to update to.') @CommandArgument('--repo', - help='Repository url to pull a snapshot from. ' - 'Supports github and googlesource.') + help='Repository url to pull a snapshot from. Supports github and googlesource.') @CommandArgument('--ignore-modified', action='store_true', - help='Ignore modified files in current checkout', - default=False) + help='Ignore modified files in current checkout', + default=False) def vendor_aom(self, **kwargs): from mozbuild.vendor_aom import VendorAOM vendor_command = self._spawn(VendorAOM) vendor_command.vendor(**kwargs) - @SubCommand('vendor', 'dav1d', description='Vendor dav1d implementation of AV1 into the source repository.') @CommandArgument('-r', '--revision', - help='Repository tag or commit to update to.') + help='Repository tag or commit to update to.') @CommandArgument('--repo', - help='Repository url to pull a snapshot from. Supports gitlab.') + help='Repository url to pull a snapshot from. Supports gitlab.') @CommandArgument('--ignore-modified', action='store_true', - help='Ignore modified files in current checkout', - default=False) + help='Ignore modified files in current checkout', + default=False) def vendor_dav1d(self, **kwargs): from mozbuild.vendor_dav1d import VendorDav1d vendor_command = self._spawn(VendorDav1d) @@ -3742,13 +3648,9 @@ class Vendor(MachCommandBase): @SubCommand('vendor', 'python', description='Vendor Python packages from pypi.org into third_party/python') @CommandArgument('--with-windows-wheel', action='store_true', - help='Vendor a wheel for Windows along with the source package', - default=False) - @CommandArgument('packages', default=None, nargs='*', - help='Packages to vendor. If omitted, packages and their dependencies ' - 'defined in Pipfile.lock will be vendored. If Pipfile has been modified, ' - 'then Pipfile.lock will be regenerated. Note that transient dependencies ' - 'may be updated when running this command.') + help='Vendor a wheel for Windows along with the source package', + default=False) + @CommandArgument('packages', default=None, nargs='*', help='Packages to vendor. If omitted, packages and their dependencies defined in Pipfile.lock will be vendored. If Pipfile has been modified, then Pipfile.lock will be regenerated. Note that transient dependencies may be updated when running this command.') def vendor_python(self, **kwargs): from mozbuild.vendor_python import VendorPython vendor_command = self._spawn(VendorPython) @@ -3766,25 +3668,21 @@ class Vendor(MachCommandBase): from mozbuild.vendor_manifest import verify_manifests verify_manifests(files) - @CommandProvider class WebRTCGTestCommands(GTestCommands): @Command('webrtc-gtest', category='testing', - description='Run WebRTC.org GTest unit tests.') + description='Run WebRTC.org GTest unit tests.') @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', - help="test_filter is a ':'-separated list of wildcard patterns " - "(called the positive patterns), optionally followed by a '-' and " - "another ':'-separated pattern list (called the negative patterns).") + help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," + "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") @CommandArgumentGroup('debugging') @CommandArgument('--debug', action='store_true', group='debugging', - help='Enable the debugger. Not specifying a --debugger option will ' - 'result in the default debugger being used.') + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') @CommandArgument('--debugger', default=None, type=str, group='debugging', - help='Name of debugger to use.') + help='Name of debugger to use.') @CommandArgument('--debugger-args', default=None, metavar='params', type=str, - group='debugging', - help='Command-line arguments to pass to the debugger itself; ' - 'split as the Bourne shell would.') + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') def gtest(self, gtest_filter, debug, debugger, debugger_args): app_path = self.get_binary_path('webrtc-gtest') @@ -3816,7 +3714,6 @@ class WebRTCGTestCommands(GTestCommands): ensure_exit_code=False, pass_thru=True) - @CommandProvider class Repackage(MachCommandBase): '''Repackages artifacts into different formats. @@ -3833,9 +3730,9 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'dmg', description='Repackage a tar file into a .dmg for OSX') @CommandArgument('--input', '-i', type=str, required=True, - help='Input filename') + help='Input filename') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') def repackage_dmg(self, input, output): if not os.path.exists(input): print('Input file does not exist: %s' % input) @@ -3852,19 +3749,19 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'installer', description='Repackage into a Windows installer exe') @CommandArgument('--tag', type=str, required=True, - help='The .tag file used to build the installer') + help='The .tag file used to build the installer') @CommandArgument('--setupexe', type=str, required=True, - help='setup.exe file inside the installer') + help='setup.exe file inside the installer') @CommandArgument('--package', type=str, required=False, - help='Optional package .zip for building a full installer') + help='Optional package .zip for building a full installer') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') @CommandArgument('--package-name', type=str, required=False, - help='Name of the package being rebuilt') + help='Name of the package being rebuilt') @CommandArgument('--sfx-stub', type=str, required=True, - help='Path to the self-extraction stub.') + help='Path to the self-extraction stub.') @CommandArgument('--use-upx', required=False, action='store_true', - help='Run UPX on the self-extraction stub.') + help='Run UPX on the self-extraction stub.') def repackage_installer(self, tag, setupexe, package, output, package_name, sfx_stub, use_upx): from mozbuild.repackaging.installer import repackage_installer repackage_installer( @@ -3881,21 +3778,21 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'msi', description='Repackage into a MSI') @CommandArgument('--wsx', type=str, required=True, - help='The wsx file used to build the installer') + help='The wsx file used to build the installer') @CommandArgument('--version', type=str, required=True, - help='The Firefox version used to create the installer') + help='The Firefox version used to create the installer') @CommandArgument('--locale', type=str, required=True, - help='The locale of the installer') + help='The locale of the installer') @CommandArgument('--arch', type=str, required=True, - help='The archtecture you are building.') + help='The archtecture you are building.') @CommandArgument('--setupexe', type=str, required=True, - help='setup.exe installer') + help='setup.exe installer') @CommandArgument('--candle', type=str, required=False, - help='location of candle binary') + help='location of candle binary') @CommandArgument('--light', type=str, required=False, - help='location of light binary') + help='location of light binary') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') def repackage_msi(self, wsx, version, locale, arch, setupexe, candle, light, output): from mozbuild.repackaging.msi import repackage_msi repackage_msi( @@ -3913,33 +3810,32 @@ class Repackage(MachCommandBase): @SubCommand('repackage', 'mar', description='Repackage into complete MAR file') @CommandArgument('--input', '-i', type=str, required=True, - help='Input filename') + help='Input filename') @CommandArgument('--mar', type=str, required=True, - help='Mar binary path') + help='Mar binary path') @CommandArgument('--output', '-o', type=str, required=True, - help='Output filename') + help='Output filename') @CommandArgument('--format', type=str, default='lzma', - choices=('lzma', 'bz2'), - help='Mar format') + choices=('lzma', 'bz2'), + help='Mar format') @CommandArgument('--arch', type=str, required=True, - help='The archtecture you are building.') + help='The archtecture you are building.') def repackage_mar(self, input, mar, output, format, arch): from mozbuild.repackaging.mar import repackage_mar repackage_mar(self.topsrcdir, input, mar, output, format, arch=arch) - @CommandProvider class Analyze(MachCommandBase): """ Get information about a file in the build graph """ @Command('analyze', category='misc', - description='Analyze the build graph.') + description='Analyze the build graph.') def analyze(self): print("Usage: ./mach analyze [files|report] [args...]") @SubCommand('analyze', 'files', - description='Get incremental build cost for file(s) from the tup database.') + description='Get incremental build cost for file(s) from the tup database.') @CommandArgument('--path', help='Path to tup db', - default=None) + default=None) @CommandArgument('files', nargs='*', help='Files to analyze') def analyze_files(self, path, files): from mozbuild.analyze.graph import Graph @@ -3955,17 +3851,16 @@ class Analyze(MachCommandBase): return 1 @SubCommand('analyze', 'all', - description='Get a report of files changed within the last n days and ' - 'their corresponding build cost.') + description='Get a report of files changed within the last n days and their corresponding build cost.') @CommandArgument('--days', '-d', type=int, default=14, - help='Number of days to include in the report.') + help='Number of days to include in the report.') @CommandArgument('--format', default='pretty', - choices=['pretty', 'csv', 'json', 'html'], - help='Print or export data in the given format.') + choices=['pretty', 'csv', 'json', 'html'], + help='Print or export data in the given format.') @CommandArgument('--limit', type=int, default=None, - help='Get the top n most expensive files from the report.') + help='Get the top n most expensive files from the report.') @CommandArgument('--path', help='Path to cost_dict.gz', - default=None) + default=None) def analyze_report(self, days, format, limit, path): from mozbuild.analyze.hg import Report self._activate_virtualenv() @@ -4074,8 +3969,7 @@ class L10NCommands(MachCommandBase): self.log(logging.INFO, 'package-multi-locale', {}, 'Invoking `mach android archive-geckoview`') self.run_process( - [mozpath.join(self.topsrcdir, 'mach'), 'android', - 'archive-geckoview'.format(locale)], + [mozpath.join(self.topsrcdir, 'mach'), 'android', 'archive-geckoview'.format(locale)], append_env=append_env, pass_thru=True, ensure_exit_code=True, diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py index 8c0ac33e82fa..fcd45bed23aa 100644 --- a/python/mozbuild/mozbuild/makeutil.py +++ b/python/mozbuild/mozbuild/makeutil.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import re @@ -62,7 +62,6 @@ class _SimpleOrderedSet(object): It doesn't expose a complete API, and normalizes path separators at insertion. ''' - def __init__(self): self._list = [] self._set = set() @@ -96,7 +95,6 @@ class Rule(object): command2 ... ''' - def __init__(self, targets=[]): self._targets = _SimpleOrderedSet() self._dependencies = _SimpleOrderedSet() @@ -129,7 +127,7 @@ class Rule(object): def dependencies(self): '''Return an iterator on the rule dependencies.''' - return iter(d for d in self._dependencies if d not in self._targets) + return iter(d for d in self._dependencies if not d in self._targets) def commands(self): '''Return an iterator on the rule commands.''' @@ -177,7 +175,6 @@ def read_dep_makefile(fh): if rule: raise Exception('Makefile finishes with a backslash. Expected more input.') - def write_dep_makefile(fh, target, deps): ''' Write a Makefile containing only target's dependencies to the file handle diff --git a/python/mozbuild/mozbuild/moz_yaml.py b/python/mozbuild/mozbuild/moz_yaml.py index 2ddf7674a351..c8805e1f9bee 100644 --- a/python/mozbuild/mozbuild/moz_yaml.py +++ b/python/mozbuild/mozbuild/moz_yaml.py @@ -306,7 +306,6 @@ def _schema_1_additional(filename, manifest, require_license_file=True): class License(object): """Voluptuous validator which verifies the license(s) are valid as per our whitelist.""" - def __call__(self, values): if isinstance(values, str): values = [values] diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py index 48faba2e8f48..2eb8d74615fa 100644 --- a/python/mozbuild/mozbuild/mozconfig.py +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import filecmp import os @@ -11,6 +11,7 @@ import sys import subprocess import traceback +from collections import defaultdict from mozpack import path as mozpath @@ -65,7 +66,7 @@ class MozconfigLoader(object): \s* [?:]?= \s* # Assignment operator surrounded by optional # spaces (?P.*$)''', # Everything else (likely the value) - re.VERBOSE) + re.VERBOSE) # Default mozconfig files in the topsrcdir. DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig') @@ -144,7 +145,7 @@ class MozconfigLoader(object): 'does not exist in any of ' + ', '.join(potential_roots)) env_path = os.path.join(existing[0], env_path) - elif not os.path.exists(env_path): # non-relative path + elif not os.path.exists(env_path): # non-relative path raise MozconfigFindException( 'MOZCONFIG environment variable refers to a path that ' 'does not exist: ' + env_path) @@ -155,12 +156,12 @@ class MozconfigLoader(object): 'non-file: ' + env_path) srcdir_paths = [os.path.join(self.topsrcdir, p) for p in - self.DEFAULT_TOPSRCDIR_PATHS] + self.DEFAULT_TOPSRCDIR_PATHS] existing = [p for p in srcdir_paths if os.path.isfile(p)] if env_path is None and len(existing) > 1: raise MozconfigFindException('Multiple default mozconfig files ' - 'present. Remove all but one. ' + ', '.join(existing)) + 'present. Remove all but one. ' + ', '.join(existing)) path = None @@ -174,12 +175,12 @@ class MozconfigLoader(object): return os.path.abspath(path) deprecated_paths = [os.path.join(self.topsrcdir, s) for s in - self.DEPRECATED_TOPSRCDIR_PATHS] + self.DEPRECATED_TOPSRCDIR_PATHS] home = env.get('HOME', None) if home is not None: deprecated_paths.extend([os.path.join(home, s) for s in - self.DEPRECATED_HOME_PATHS]) + self.DEPRECATED_HOME_PATHS]) for path in deprecated_paths: if os.path.exists(path): @@ -242,7 +243,7 @@ class MozconfigLoader(object): # We need to capture stderr because that's where the shell sends # errors if execution fails. output = subprocess.check_output(command, stderr=subprocess.STDOUT, - cwd=self.topsrcdir, env=env) + cwd=self.topsrcdir, env=env) except subprocess.CalledProcessError as e: lines = e.output.splitlines() @@ -305,7 +306,7 @@ class MozconfigLoader(object): # Environment variables also appear as shell variables, but that's # uninteresting duplication of information. Filter them out. - def filt(x, y): return {k: v for k, v in x.items() if k not in y} + filt = lambda x, y: {k: v for k, v in x.items() if k not in y} result['vars'] = diff_vars( filt(parsed['vars_before'], parsed['env_before']), filt(parsed['vars_after'], parsed['env_after']) diff --git a/python/mozbuild/mozbuild/mozinfo.py b/python/mozbuild/mozbuild/mozinfo.py index a5b301943bf9..e74eef261da9 100755 --- a/python/mozbuild/mozbuild/mozinfo.py +++ b/python/mozbuild/mozbuild/mozinfo.py @@ -5,7 +5,7 @@ # This module produces a JSON file that provides basic build info and # configuration metadata. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import re @@ -58,7 +58,7 @@ def build_dict(config, env=os.environ): # processor p = substs["TARGET_CPU"] # do some slight massaging for some values - # TODO: retain specific values in case someone wants them? + #TODO: retain specific values in case someone wants them? if p.startswith("arm"): p = "arm" elif re.match("i[3-9]86", p): @@ -130,7 +130,7 @@ def build_dict(config, env=os.environ): d['platform_guess'] = guess_platform() d['buildtype_guess'] = guess_buildtype() - if d.get('buildapp', '') == 'mobile/android' and 'MOZ_ANDROID_MIN_SDK_VERSION' in substs: + if 'buildapp' in d and d['buildapp'] == 'mobile/android' and 'MOZ_ANDROID_MIN_SDK_VERSION' in substs: d['android_min_sdk'] = substs['MOZ_ANDROID_MIN_SDK_VERSION'] return d diff --git a/python/mozbuild/mozbuild/nodeutil.py b/python/mozbuild/mozbuild/nodeutil.py index 2e7bd067754b..85e11d3b8b04 100644 --- a/python/mozbuild/mozbuild/nodeutil.py +++ b/python/mozbuild/mozbuild/nodeutil.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import subprocess diff --git a/python/mozbuild/mozbuild/preprocessor.py b/python/mozbuild/mozbuild/preprocessor.py index 0b4f51a4e538..200651712ab5 100644 --- a/python/mozbuild/mozbuild/preprocessor.py +++ b/python/mozbuild/mozbuild/preprocessor.py @@ -22,14 +22,12 @@ value : | \w+ # string identifier or value; """ -from __future__ import absolute_import, print_function - import sys import os import re from optparse import OptionParser import errno -from mozbuild.makeutil import Makefile +from makeutil import Makefile # hack around win32 mangling our line endings # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443 @@ -200,7 +198,6 @@ class Expression: rv = not rv return rv # Helper function to evaluate __get_logical_and and __get_logical_or results - def eval_logical_op(tok): left = opmap[tok[0].type](tok[0]) right = opmap[tok[2].type](tok[2]) @@ -220,13 +217,12 @@ class Expression: 'defined': lambda tok: tok.value in context, 'int': lambda tok: tok.value} - return opmap[self.e.type](self.e) + return opmap[self.e.type](self.e); class __AST(list): """ Internal class implementing Abstract Syntax Tree nodes """ - def __init__(self, type): self.type = type super(self.__class__, self).__init__(self) @@ -235,14 +231,11 @@ class Expression: """ Internal class implementing Abstract Syntax Tree leafs """ - def __init__(self, type, value): self.value = value self.type = type - def __str__(self): return self.value.__str__() - def __repr__(self): return self.value.__repr__() @@ -252,16 +245,13 @@ class Expression: It has two members, offset and content, which give the offset of the error and the offending content. """ - def __init__(self, expression): self.offset = expression.offset self.content = expression.content[:3] - def __str__(self): return 'Unexpected content at offset {0}, "{1}"'.format(self.offset, self.content) - class Context(dict): """ This class holds variable values by subclassing dict, and while it @@ -276,7 +266,6 @@ class Context(dict): to reflect the ambiguity between string literals and preprocessor variables. """ - def __getitem__(self, key): if key in self: return super(self.__class__, self).__getitem__(key) @@ -296,9 +285,9 @@ class Preprocessor: def __init__(self, defines=None, marker='#'): self.context = Context() - for k, v in {'FILE': '', - 'LINE': 0, - 'DIRECTORY': os.path.abspath('.')}.iteritems(): + for k,v in {'FILE': '', + 'LINE': 0, + 'DIRECTORY': os.path.abspath('.')}.iteritems(): self.context[k] = v try: # Can import globally because of bootstrapping issues. @@ -352,8 +341,7 @@ class Preprocessor: elif self.actionLevel == 1: msg = 'no useful preprocessor directives found' if msg: - class Fake(object): - pass + class Fake(object): pass fake = Fake() fake.context = { 'FILE': file, @@ -466,7 +454,7 @@ class Preprocessor: self.actionLevel = 2 self.out.write(filteredLine) - def handleCommandLine(self, args, defaultToStdin=False): + def handleCommandLine(self, args, defaultToStdin = False): """ Parse a commandline into this parser. Uses OptionParser internally, no args mean sys.argv[1:]. @@ -497,6 +485,11 @@ class Preprocessor: if not options.output: raise Preprocessor.Error(self, "--depend doesn't work with stdout", None) + try: + from makeutil import Makefile + except: + raise Preprocessor.Error(self, "--depend requires the " + "mozbuild.makeutil module", None) depfile = get_output_file(options.depend) if args: @@ -512,10 +505,9 @@ class Preprocessor: if options.output: out.close() - def getCommandLineParser(self, unescapeDefines=False): + def getCommandLineParser(self, unescapeDefines = False): escapedValue = re.compile('".*"$') numberValue = re.compile('\d+$') - def handleD(option, opt, value, parser): vals = value.split('=', 1) if len(vals) == 1: @@ -526,16 +518,12 @@ class Preprocessor: elif numberValue.match(vals[1]): vals[1] = int(vals[1]) self.context[vals[0]] = vals[1] - def handleU(option, opt, value, parser): del self.context[value] - def handleF(option, opt, value, parser): self.do_filter(value) - def handleMarker(option, opt, value, parser): self.setMarker(value) - def handleSilenceDirectiveWarnings(option, opt, value, parse): self.setSilenceDirectiveWarnings(True) p = OptionParser() @@ -546,7 +534,7 @@ class Preprocessor: p.add_option('-F', action='callback', callback=handleF, type="string", metavar="FILTER", help='Enable the specified filter') p.add_option('-o', '--output', type="string", default=None, - metavar="FILENAME", help='Output to the specified file ' + + metavar="FILENAME", help='Output to the specified file '+ 'instead of stdout') p.add_option('--depend', type="string", default=None, metavar="FILENAME", help='Generate dependencies in the given file') @@ -595,10 +583,9 @@ class Preprocessor: val = self.applyFilters(m.group('value')) try: val = int(val) - except Exception: + except: pass self.context[m.group('name')] = val - def do_undef(self, args): m = re.match('(?P\w+)$', args, re.U) if not m: @@ -606,11 +593,9 @@ class Preprocessor: if args in self.context: del self.context[args] # Logic - def ensure_not_else(self): if len(self.ifStates) == 0 or self.ifStates[-1] == 2: sys.stderr.write('WARNING: bad nesting of #else in %s\n' % self.context['FILE']) - def do_if(self, args, replace=False): if self.disableLevel and not replace: self.disableLevel += 1 @@ -634,7 +619,6 @@ class Preprocessor: else: self.ifStates.append(self.disableLevel) pass - def do_ifdef(self, args, replace=False): if self.disableLevel and not replace: self.disableLevel += 1 @@ -650,7 +634,6 @@ class Preprocessor: else: self.ifStates.append(self.disableLevel) pass - def do_ifndef(self, args, replace=False): if self.disableLevel and not replace: self.disableLevel += 1 @@ -666,59 +649,51 @@ class Preprocessor: else: self.ifStates.append(self.disableLevel) pass - - def do_else(self, args, ifState=2): + def do_else(self, args, ifState = 2): self.ensure_not_else() hadTrue = self.ifStates[-1] == 0 - self.ifStates[-1] = ifState # in-else + self.ifStates[-1] = ifState # in-else if hadTrue: self.disableLevel = 1 return self.disableLevel = 0 - def do_elif(self, args): if self.disableLevel == 1: if self.ifStates[-1] == 1: self.do_if(args, replace=True) else: self.do_else(None, self.ifStates[-1]) - def do_elifdef(self, args): if self.disableLevel == 1: if self.ifStates[-1] == 1: self.do_ifdef(args, replace=True) else: self.do_else(None, self.ifStates[-1]) - def do_elifndef(self, args): if self.disableLevel == 1: if self.ifStates[-1] == 1: self.do_ifndef(args, replace=True) else: self.do_else(None, self.ifStates[-1]) - def do_endif(self, args): if self.disableLevel > 0: self.disableLevel -= 1 if self.disableLevel == 0: self.ifStates.pop() # output processing - def do_expand(self, args): lst = re.split('__(\w+)__', args, re.U) - + do_replace = False def vsubst(v): if v in self.context: return str(self.context[v]) return '' for i in range(1, len(lst), 2): lst[i] = vsubst(lst[i]) - lst.append('\n') # add back the newline + lst.append('\n') # add back the newline self.write(reduce(lambda x, y: x+y, lst, '')) - def do_literal(self, args): self.write(args + '\n') - def do_filter(self, args): filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)] if len(filters) == 0: @@ -730,7 +705,6 @@ class Preprocessor: filterNames.sort() self.filters = [(fn, current[fn]) for fn in filterNames] return - def do_unfilter(self, args): filters = args.split(' ') current = dict(self.filters) @@ -745,14 +719,12 @@ class Preprocessor: # # emptyLines # Strips blank lines from the output. - def filter_emptyLines(self, aLine): if aLine == '\n': return '' return aLine # slashslash # Strips everything after // - def filter_slashslash(self, aLine): if (aLine.find('//') == -1): return aLine @@ -762,12 +734,10 @@ class Preprocessor: return aLine # spaces # Collapses sequences of spaces into a single space - def filter_spaces(self, aLine): return re.sub(' +', ' ', aLine).strip(' ') # substition # helper to be used by both substition and attemptSubstitution - def filter_substitution(self, aLine, fatal=True): def repl(matchobj): varname = matchobj.group('VAR') @@ -777,11 +747,9 @@ class Preprocessor: raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname) return matchobj.group(0) return self.varsubst.sub(repl, aLine) - def filter_attemptSubstitution(self, aLine): return self.filter_substitution(aLine, fatal=False) # File ops - def do_include(self, args, filters=True): """ Preprocess a given file. @@ -801,7 +769,7 @@ class Preprocessor: args = open(args, 'rU') except Preprocessor.Error: raise - except Exception: + except: raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args)) self.checkLineNumbers = bool(re.search('\.(js|jsm|java|webidl)(?:\.in)?$', args.name)) oldFile = self.context['FILE'] @@ -838,17 +806,15 @@ class Preprocessor: self.context['LINE'] = oldLine self.context['DIRECTORY'] = oldDir self.curdir = oldCurdir - def do_includesubst(self, args): args = self.filter_substitution(args) self.do_include(args) - def do_error(self, args): raise Preprocessor.Error(self, 'Error: ', str(args)) def preprocess(includes=[sys.stdin], defines={}, - output=sys.stdout, + output = sys.stdout, marker='#'): pp = Preprocessor(defines=defines, marker=marker) diff --git a/python/mozbuild/mozbuild/pythonutil.py b/python/mozbuild/mozbuild/pythonutil.py index 21cfd547178b..1d959f800cde 100644 --- a/python/mozbuild/mozbuild/pythonutil.py +++ b/python/mozbuild/mozbuild/pythonutil.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import subprocess diff --git a/python/mozbuild/mozbuild/repackaging/application_ini.py b/python/mozbuild/mozbuild/repackaging/application_ini.py index 0d03a0e59b1b..5c975b154207 100644 --- a/python/mozbuild/mozbuild/repackaging/application_ini.py +++ b/python/mozbuild/mozbuild/repackaging/application_ini.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import ConfigParser import mozpack.path as mozpath diff --git a/python/mozbuild/mozbuild/repackaging/dmg.py b/python/mozbuild/mozbuild/repackaging/dmg.py index a80fcab8df6c..9868e85697b1 100644 --- a/python/mozbuild/mozbuild/repackaging/dmg.py +++ b/python/mozbuild/mozbuild/repackaging/dmg.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import errno import os import tempfile @@ -11,8 +9,7 @@ import tarfile import shutil import mozpack.path as mozpath from mozpack.dmg import create_dmg -from mozbuild.repackaging.application_ini import get_application_ini_value - +from application_ini import get_application_ini_value def repackage_dmg(infile, output): diff --git a/python/mozbuild/mozbuild/repackaging/installer.py b/python/mozbuild/mozbuild/repackaging/installer.py index 43db1d068bb1..ec891f4f426e 100644 --- a/python/mozbuild/mozbuild/repackaging/installer.py +++ b/python/mozbuild/mozbuild/repackaging/installer.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import tempfile import shutil @@ -13,8 +11,7 @@ from mozbuild.action.exe_7z_archive import archive_exe from mozbuild.util import ensureParentDir -def repackage_installer(topsrcdir, tag, setupexe, package, output, - package_name, sfx_stub, use_upx): +def repackage_installer(topsrcdir, tag, setupexe, package, output, package_name, sfx_stub, use_upx): if package and not zipfile.is_zipfile(package): raise Exception("Package file %s is not a valid .zip file." % package) if package is not None and package_name is None: diff --git a/python/mozbuild/mozbuild/repackaging/mar.py b/python/mozbuild/mozbuild/repackaging/mar.py index 08d058936e46..6ddac41c8a0a 100644 --- a/python/mozbuild/mozbuild/repackaging/mar.py +++ b/python/mozbuild/mozbuild/repackaging/mar.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import sys import tempfile @@ -12,7 +10,7 @@ import zipfile import tarfile import subprocess import mozpack.path as mozpath -from mozbuild.repackaging.application_ini import get_application_ini_value +from application_ini import get_application_ini_value from mozbuild.util import ensureParentDir diff --git a/python/mozbuild/mozbuild/repackaging/msi.py b/python/mozbuild/mozbuild/repackaging/msi.py index 93d800cd44a9..14228c83c96a 100644 --- a/python/mozbuild/mozbuild/repackaging/msi.py +++ b/python/mozbuild/mozbuild/repackaging/msi.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import tempfile import shutil @@ -18,17 +16,16 @@ _MSI_ARCH = { 'x86_64': 'x64', } - def update_wsx(wfile, pvalues): parsed = minidom.parse(wfile) # construct a dictinary for the pre-processing options # iterate over that list and add them to the wsx xml doc - for k, v in pvalues.items(): - entry = parsed.createProcessingInstruction('define', k + ' = "' + v + '"') - root = parsed.firstChild - parsed.insertBefore(entry, root) + for k,v in pvalues.items(): + entry = parsed.createProcessingInstruction('define', k + ' = "' + v + '"') + root = parsed.firstChild + parsed.insertBefore(entry, root) # write out xml to new wfile new_w_file = wfile + ".new" fh = open(new_w_file, "wb") @@ -59,7 +56,7 @@ def repackage_msi(topsrcdir, wsx, version, locale, arch, setupexe, candle, light raise Exception("%s does not exist." % light) embeddedVersion = '0.0.0.0' # Version string cannot contain 'a' or 'b' when embedding in msi manifest. - if 'a' not in version and 'b' not in version: + if not 'a' in version and not 'b' in version: if version.endswith('esr'): parts = version[:-3].split('.') else: @@ -79,8 +76,8 @@ def repackage_msi(topsrcdir, wsx, version, locale, arch, setupexe, candle, light try: wsx_file = os.path.split(wsx)[1] shutil.copy(wsx, tmpdir) - temp_wsx_file = os.path.join(tmpdir, wsx_file) - temp_wsx_file = mozpath.realpath(temp_wsx_file) + temp_wsx_file = os.path.join(tmpdir, wsx_file) + temp_wsx_file = mozpath.realpath(temp_wsx_file) pre_values = {'Vendor': 'Mozilla', 'BrandFullName': 'Mozilla Firefox', 'Version': version, @@ -103,7 +100,7 @@ def repackage_msi(topsrcdir, wsx, version, locale, arch, setupexe, candle, light '-out', wix_installer, wix_object_file] subprocess.check_call(light_cmd, env=env) os.remove(wix_object_file) - # mv file to output dir + #mv file to output dir shutil.move(wix_installer, output) finally: os.chdir(old_cwd) diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/shellutil.py index 140feecb2c31..eaa9510605cb 100644 --- a/python/mozbuild/mozbuild/shellutil.py +++ b/python/mozbuild/mozbuild/shellutil.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import re @@ -24,7 +22,6 @@ def _tokens2re(**tokens): # backslash, captured in the "escape" match group. return re.compile('(?:%s|%s)' % (nonescaped, r'(?P\\\\)')) - UNQUOTED_TOKENS_RE = _tokens2re( whitespace=r'[\t\r\n ]+', quote=r'[\'"]', @@ -57,7 +54,6 @@ class _ClineSplitter(object): Parses a given command line string and creates a list of command and arguments, with wildcard expansion. ''' - def __init__(self, cline): self.arg = None self.cline = cline diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py index 9270270ea8e9..2d9032ec0c5e 100644 --- a/python/mozbuild/mozbuild/sphinx.py +++ b/python/mozbuild/mozbuild/sphinx.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import importlib import os @@ -187,9 +187,9 @@ def setup(app): # properly. We leverage the in-tree virtualenv for this. topsrcdir = manager.topsrcdir ve = VirtualenvManager(topsrcdir, - os.path.join(topsrcdir, 'dummy-objdir'), - os.path.join(app.outdir, '_venv'), - sys.stderr, - os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt')) + os.path.join(topsrcdir, 'dummy-objdir'), + os.path.join(app.outdir, '_venv'), + sys.stderr, + os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt')) ve.ensure() ve.activate() diff --git a/python/mozbuild/mozbuild/telemetry.py b/python/mozbuild/mozbuild/telemetry.py index ceb2a31e28af..49bd398d84ec 100644 --- a/python/mozbuild/mozbuild/telemetry.py +++ b/python/mozbuild/mozbuild/telemetry.py @@ -238,7 +238,6 @@ def get_build_attrs(attrs): res['cpu_percent'] = int(round(usage['cpu_percent'])) return res - def filter_args(command, argv, paths): ''' Given the full list of command-line arguments, remove anything up to and including `command`, diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/mozbuild/mozbuild/test/action/test_buildlist.py index a48afb4455ac..9c2631812d41 100644 --- a/python/mozbuild/mozbuild/test/action/test_buildlist.py +++ b/python/mozbuild/mozbuild/test/action/test_buildlist.py @@ -2,12 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest -import os -import os.path +import os, sys, os.path, time from tempfile import mkdtemp from shutil import rmtree import mozunit @@ -16,79 +13,77 @@ from mozbuild.action.buildlist import addEntriesToListFile class TestBuildList(unittest.TestCase): - """ - Unit tests for buildlist.py - """ + """ + Unit tests for buildlist.py + """ + def setUp(self): + self.tmpdir = mkdtemp() - def setUp(self): - self.tmpdir = mkdtemp() + def tearDown(self): + rmtree(self.tmpdir) - def tearDown(self): - rmtree(self.tmpdir) + # utility methods for tests + def touch(self, file, dir=None): + if dir is None: + dir = self.tmpdir + f = os.path.join(dir, file) + open(f, 'w').close() + return f - # utility methods for tests - def touch(self, file, dir=None): - if dir is None: - dir = self.tmpdir - f = os.path.join(dir, file) - open(f, 'w').close() - return f + def assertFileContains(self, filename, l): + """Assert that the lines in the file |filename| are equal + to the contents of the list |l|, in order.""" + l = l[:] + f = open(filename, 'r') + lines = [line.rstrip() for line in f.readlines()] + f.close() + for line in lines: + self.assert_(len(l) > 0, + "ran out of expected lines! (expected '{0}', got '{1}')" + .format(l, lines)) + self.assertEqual(line, l.pop(0)) + self.assert_(len(l) == 0, + "not enough lines in file! (expected '{0}'," + " got '{1}'".format(l, lines)) - def assertFileContains(self, filename, l): - """Assert that the lines in the file |filename| are equal - to the contents of the list |l|, in order.""" - l = l[:] - f = open(filename, 'r') - lines = [line.rstrip() for line in f.readlines()] - f.close() - for line in lines: - self.assert_(len(l) > 0, - "ran out of expected lines! (expected '{0}', got '{1}')" - .format(l, lines)) - self.assertEqual(line, l.pop(0)) - self.assert_(len(l) == 0, - "not enough lines in file! (expected '{0}'," - " got '{1}'".format(l, lines)) + def test_basic(self): + "Test that addEntriesToListFile works when file doesn't exist." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + # ensure that attempting to add the same entries again doesn't change it + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) - def test_basic(self): - "Test that addEntriesToListFile works when file doesn't exist." - testfile = os.path.join(self.tmpdir, "test.list") - l = ["a", "b", "c"] - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) - # ensure that attempting to add the same entries again doesn't change it - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) + def test_append(self): + "Test adding new entries." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + l2 = ["x","y","z"] + addEntriesToListFile(testfile, l2) + l.extend(l2) + self.assertFileContains(testfile, l) - def test_append(self): - "Test adding new entries." - testfile = os.path.join(self.tmpdir, "test.list") - l = ["a", "b", "c"] - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) - l2 = ["x", "y", "z"] - addEntriesToListFile(testfile, l2) - l.extend(l2) - self.assertFileContains(testfile, l) - - def test_append_some(self): - "Test adding new entries mixed with existing entries." - testfile = os.path.join(self.tmpdir, "test.list") - l = ["a", "b", "c"] - addEntriesToListFile(testfile, l) - self.assertFileContains(testfile, l) - addEntriesToListFile(testfile, ["a", "x", "c", "z"]) - self.assertFileContains(testfile, ["a", "b", "c", "x", "z"]) - - def test_add_multiple(self): - """Test that attempting to add the same entry multiple times results in - only one entry being added.""" - testfile = os.path.join(self.tmpdir, "test.list") - addEntriesToListFile(testfile, ["a", "b", "a", "a", "b"]) - self.assertFileContains(testfile, ["a", "b"]) - addEntriesToListFile(testfile, ["c", "a", "c", "b", "c"]) - self.assertFileContains(testfile, ["a", "b", "c"]) + def test_append_some(self): + "Test adding new entries mixed with existing entries." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + addEntriesToListFile(testfile, ["a", "x", "c", "z"]) + self.assertFileContains(testfile, ["a", "b", "c", "x", "z"]) + def test_add_multiple(self): + """Test that attempting to add the same entry multiple times results in + only one entry being added.""" + testfile = os.path.join(self.tmpdir, "test.list") + addEntriesToListFile(testfile, ["a","b","a","a","b"]) + self.assertFileContains(testfile, ["a","b"]) + addEntriesToListFile(testfile, ["c","a","c","b","c"]) + self.assertFileContains(testfile, ["a","b","c"]) if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/action/test_langpack_manifest.py b/python/mozbuild/mozbuild/test/action/test_langpack_manifest.py index 4966d9851392..328a8440b62b 100644 --- a/python/mozbuild/mozbuild/test/action/test_langpack_manifest.py +++ b/python/mozbuild/mozbuild/test/action/test_langpack_manifest.py @@ -3,8 +3,6 @@ # Any copyright is dedicated to the Public Domain. # http://creativecommons.org/publicdomain/zero/1.0/ -from __future__ import absolute_import, print_function - import unittest import json diff --git a/python/mozbuild/mozbuild/test/action/test_node.py b/python/mozbuild/mozbuild/test/action/test_node.py index aabd741b2b21..0e4323a91f46 100644 --- a/python/mozbuild/mozbuild/test/action/test_node.py +++ b/python/mozbuild/mozbuild/test/action/test_node.py @@ -21,7 +21,6 @@ test_data_path = mozpath.join(test_data_path, 'data', 'node') def data(name): return os.path.join(test_data_path, name) - TEST_SCRIPT = data("node-test-script.js") NONEXISTENT_TEST_SCRIPT = data("non-existent-test-script.js") @@ -37,7 +36,7 @@ class TestNode(unittest.TestCase): def tearDown(self): try: SCRIPT_ALLOWLIST.remove(TEST_SCRIPT) - except Exception: + except: pass def test_generate_no_returned_deps(self): diff --git a/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py b/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py index 752a7a4ea2c0..8cefc76938b3 100644 --- a/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py +++ b/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py @@ -3,7 +3,7 @@ # Any copyright is dedicated to the Public Domain. # http://creativecommons.org/publicdomain/zero/1.0/ -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import unittest diff --git a/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py b/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py index 0a082cdfc921..a912e3be29b9 100644 --- a/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py +++ b/python/mozbuild/mozbuild/test/action/test_process_install_manifest.py @@ -3,14 +3,19 @@ # Any copyright is dedicated to the Public Domain. # http://creativecommons.org/publicdomain/zero/1.0/ -from __future__ import absolute_import, print_function - import os import mozunit +from unittest import expectedFailure + +from mozpack.copier import ( + FileCopier, + FileRegistry, +) from mozpack.manifests import ( InstallManifest, + UnreadableInstallManifest, ) from mozpack.test.test_files import TestWithTmpDir @@ -65,6 +70,5 @@ class TestGenerateManifest(TestWithTmpDir): self.assertFalse(os.path.exists(self.tmppath('dest/foo/file2'))) self.assertFalse(os.path.exists(self.tmppath('dest/foo/file3'))) - if __name__ == '__main__': mozunit.main() diff --git a/python/mozbuild/mozbuild/test/analyze/test_graph.py b/python/mozbuild/mozbuild/test/analyze/test_graph.py index 340629e782ea..fe9b99c5679a 100644 --- a/python/mozbuild/mozbuild/test/analyze/test_graph.py +++ b/python/mozbuild/mozbuild/test/analyze/test_graph.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest import sqlite3 as lite import mozunit @@ -22,40 +20,40 @@ CREATE_NORMAL_LINK = """CREATE TABLE normal_link (from_id integer, to_id integer, unique(from_id, to_id));""" -NODE_DATA = [(1, 0, 2, -1, '.'), - (2, 100, 0, 1, 'Base64.cpp'), - (3, 200, 0, 1, 'nsArray.cpp'), - (4, 100, 0, 1, 'nsWildCard.h'), - (5, -1, 1, 9426, 'CDD Unified_cpp_xpcom_io0.cpp'), - (6, -1, 1, 5921, 'CXX Unified_cpp_xpcom_ds0.cpp'), - (7, -1, 1, 11077, 'CXX /builds/worker/workspace/build/src/dom/\ +NODE_DATA = [(1, 0 ,2, -1, '.'), + (2, 100, 0, 1, 'Base64.cpp'), + (3, 200, 0, 1, 'nsArray.cpp'), + (4, 100, 0, 1, 'nsWildCard.h'), + (5, -1, 1, 9426, 'CDD Unified_cpp_xpcom_io0.cpp'), + (6, -1, 1, 5921, 'CXX Unified_cpp_xpcom_ds0.cpp'), + (7, -1, 1, 11077, 'CXX /builds/worker/workspace/build/src/dom/\ plugins/base/snNPAPIPlugin.cpp'), - (8, -1, 1, 7677, 'CXX Unified_cpp_xpcom_io1.cpp'), - (9, -1, 1, 8672, 'CXX Unified_cpp_modules_libjar0.cpp'), - (10, -1, 4, 1, 'Unified_cpp_xpcom_io0.o'), - (11, -1, 4, 1, 'Unified_cpp_xpcom_dso.o'), - (12, -1, 4, 1, 'nsNPAPIPlugin.o'), - (13, -1, 4, 1, 'Unified_cpp_xpcom_io1.o'), - (14, -1, 4, 1, 'Unified_cpp_modules_libjar0.o'), - (15, -1, 1, 52975, 'LINK libxul.so'), - (16, -1, 4, 1, 'libxul.so'), - (17, -1, 1, 180, 'LINK libtestcrasher.so'), - (18, -1, 1, 944, 'python /builds/worker/workspace/build/src/toolkit/\ + (8, -1, 1, 7677, 'CXX Unified_cpp_xpcom_io1.cpp'), + (9, -1, 1, 8672, 'CXX Unified_cpp_modules_libjar0.cpp'), + (10, -1, 4, 1, 'Unified_cpp_xpcom_io0.o'), + (11, -1, 4, 1, 'Unified_cpp_xpcom_dso.o'), + (12, -1, 4, 1, 'nsNPAPIPlugin.o'), + (13, -1, 4, 1, 'Unified_cpp_xpcom_io1.o'), + (14, -1, 4, 1, 'Unified_cpp_modules_libjar0.o'), + (15, -1, 1, 52975, 'LINK libxul.so'), + (16, -1, 4, 1, 'libxul.so'), + (17, -1, 1, 180, 'LINK libtestcrasher.so'), + (18, -1, 1, 944, 'python /builds/worker/workspace/build/src/toolkit/\ library/dependentlibs.py:gen_list -> [dependentlibs.list, \ dependentlibs.list.gtest, dependentlibs.list.pp]'), - (19, -1, 1, 348, 'LINK ../../dist/bin/plugin-container'), - (20, -1, 1, 342, 'LINK ../../../dist/bin/xpcshell'), - (21, -1, 4, 1, 'libtestcrasher.so'), - (22, -1, 4, 1, 'dependentlibs.list'), - (23, -1, 4, 1, 'dependentlibs.list.gtest'), - (24, -1, 4, 1, 'dependentlibs.list.pp'), - (25, -1, 4, 1, 'plugin-container'), - (26, -1, 4, 1, 'xpcshell'), - (27, -1, 6, 1, ''), - (28, 1, 0, 1, 'dummy node'), - (100, 300, 2, -1, 'io'), - (200, 300, 2, -1, 'ds'), - (300, 1, 2, -1, 'xpcom')] + (19, -1, 1, 348, 'LINK ../../dist/bin/plugin-container'), + (20, -1, 1, 342, 'LINK ../../../dist/bin/xpcshell'), + (21, -1, 4, 1, 'libtestcrasher.so'), + (22, -1, 4, 1, 'dependentlibs.list'), + (23, -1, 4, 1, 'dependentlibs.list.gtest'), + (24, -1, 4, 1, 'dependentlibs.list.pp'), + (25, -1, 4, 1, 'plugin-container'), + (26, -1, 4, 1, 'xpcshell'), + (27, -1, 6, 1, ''), + (28, 1, 0, 1, 'dummy node'), + (100, 300, 2, -1, 'io'), + (200, 300, 2, -1, 'ds'), + (300, 1, 2, -1, 'xpcom')] NORMAL_LINK_DATA = [(2, 5), (3, 6), (4, 7), (4, 8), (4, 9), (5, 10), (6, 11), (7, 12), (8, 13), (9, 14), (10, 15), (11, 15), (12, 15), @@ -65,7 +63,6 @@ NORMAL_LINK_DATA = [(2, 5), (3, 6), (4, 7), (4, 8), (4, 9), (5, 10), (6, 11), PATH_TO_TEST_DB = ':memory:' - class TestGraph(unittest.TestCase): @classmethod def setUpClass(cls): @@ -96,10 +93,10 @@ class TestGraph(unittest.TestCase): self.assertEqual(len(g.get_node(21).cmds), 0) self.assertEqual(len(g.get_node(28).cmds), 0) # one immediate command child - self.assertItemsEqual(g.get_node(2).get_cmd_ids(), [5] + libxul) - self.assertItemsEqual(g.get_node(3).get_cmd_ids(), [6] + libxul) + self.assertItemsEqual(g.get_node(2).get_cmd_ids(),[5] + libxul) + self.assertItemsEqual(g.get_node(3).get_cmd_ids(),[6] + libxul) # multiple immediate command children - self.assertItemsEqual(g.get_node(4).get_cmd_ids(), [7, 8, 9] + libxul) + self.assertItemsEqual(g.get_node(4).get_cmd_ids(),[7, 8, 9] + libxul) # node is not a file or command self.assertItemsEqual(g.get_node(16).get_cmd_ids(), libxul[1:]) self.assertItemsEqual(g.get_node(11).get_cmd_ids(), libxul) @@ -134,6 +131,5 @@ class TestGraph(unittest.TestCase): self.assertEqual(g.get_node(4).path, 'xpcom/io/nsWildCard.h') self.assertEqual(g.get_node(28).path, 'dummy node') - if __name__ == '__main__': - mozunit.main() + mozunit.main() \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py index 133874296e2f..c5959094cf27 100644 --- a/python/mozbuild/mozbuild/test/backend/common.py +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import unittest diff --git a/python/mozbuild/mozbuild/test/backend/test_build.py b/python/mozbuild/mozbuild/test/backend/test_build.py index ed9aed789277..ada6d6e75b7b 100644 --- a/python/mozbuild/mozbuild/test/backend/test_build.py +++ b/python/mozbuild/mozbuild/test/backend/test_build.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import unicode_literals, print_function import buildconfig import os @@ -58,7 +58,7 @@ class TestBuild(unittest.TestCase): backend(config).consume(definitions) yield config - except Exception: + except: raise finally: if not os.environ.get('MOZ_NO_CLEANUP'): @@ -73,7 +73,7 @@ class TestBuild(unittest.TestCase): try: yield handle_make_line - except Exception: + except: print('\n'.join(lines)) raise @@ -233,6 +233,5 @@ class TestBuild(unittest.TestCase): 'bin/app/modules/foo.jsm': 'foo.jsm\n', }) - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/backend/test_configenvironment.py b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py index 84b0a4ac0f4f..95593e1869e8 100644 --- a/python/mozbuild/mozbuild/test/backend/test_configenvironment.py +++ b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py @@ -2,11 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - -import os +import os, posixpath +from StringIO import StringIO import unittest -from mozunit import main +from mozunit import main, MockedOpen import mozbuild.backend.configenvironment as ConfigStatus @@ -19,7 +18,7 @@ class ConfigEnvironment(ConfigStatus.ConfigEnvironment): def __init__(self, *args, **kwargs): ConfigStatus.ConfigEnvironment.__init__(self, *args, **kwargs) # Be helpful to unit tests - if 'top_srcdir' not in self.substs: + if not 'top_srcdir' in self.substs: if os.path.isabs(self.topsrcdir): top_srcdir = self.topsrcdir.replace(os.sep, '/') else: @@ -40,16 +39,15 @@ class TestEnvironment(unittest.TestCase): and ALLEMPTYSUBSTS. ''' env = ConfigEnvironment('.', '.', - defines={'foo': 'bar', 'baz': 'qux 42', - 'abc': "d'e'f", 'extra': 'foobar'}, - non_global_defines=['extra', 'ignore'], - substs={'FOO': 'bar', 'FOOBAR': '', 'ABC': 'def', - 'bar': 'baz qux', 'zzz': '"abc def"', - 'qux': ''}) + defines = { 'foo': 'bar', 'baz': 'qux 42', + 'abc': "d'e'f", 'extra': 'foobar' }, + non_global_defines = ['extra', 'ignore'], + substs = { 'FOO': 'bar', 'FOOBAR': '', 'ABC': 'def', + 'bar': 'baz qux', 'zzz': '"abc def"', + 'qux': '' }) # non_global_defines should be filtered out in ACDEFINES. # Original order of the defines need to be respected in ACDEFINES - self.assertEqual(env.substs['ACDEFINES'], - """-Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar""") + self.assertEqual(env.substs['ACDEFINES'], """-Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar""") # Likewise for ALLSUBSTS, which also must contain ACDEFINES self.assertEqual(env.substs['ALLSUBSTS'], '''ABC = def ACDEFINES = -Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar diff --git a/python/mozbuild/mozbuild/test/backend/test_fastermake.py b/python/mozbuild/mozbuild/test/backend/test_fastermake.py index 311e2eb26c75..eba9110a0ab3 100644 --- a/python/mozbuild/mozbuild/test/backend/test_fastermake.py +++ b/python/mozbuild/mozbuild/test/backend/test_fastermake.py @@ -2,9 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals +import cPickle as pickle +import json import os +import unittest from mozpack.copier import FileRegistry from mozpack.manifests import InstallManifest @@ -22,15 +25,15 @@ class TestFasterMakeBackend(BackendTester): """Ensure the FasterMakeBackend works without error.""" env = self._consume('stub0', FasterMakeBackend) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.FasterMakeBackend'))) + 'backend.FasterMakeBackend'))) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.FasterMakeBackend.in'))) + 'backend.FasterMakeBackend.in'))) def test_final_target_files_wildcard(self): """Ensure that wildcards in FINAL_TARGET_FILES work properly.""" env = self._consume('final-target-files-wildcard', FasterMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, - 'faster', 'install_dist_bin')) + 'faster', 'install_dist_bin')) self.assertEqual(len(m), 1) reg = FileRegistry() m.populate_registry(reg) diff --git a/python/mozbuild/mozbuild/test/backend/test_gn_processor.py b/python/mozbuild/mozbuild/test/backend/test_gn_processor.py index 2a6b126e342d..d98766318b95 100644 --- a/python/mozbuild/mozbuild/test/backend/test_gn_processor.py +++ b/python/mozbuild/mozbuild/test/backend/test_gn_processor.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import shutil @@ -21,6 +21,7 @@ from mozbuild.gn_processor import ( GnMozbuildWriterBackend, find_common_attrs, ) +from mozbuild.backend.recursivemake import RecursiveMakeBackend from mozbuild.frontend.data import ( ComputedFlags, @@ -29,7 +30,6 @@ from mozbuild.frontend.data import ( UnifiedSources, ) - class TestGnMozbuildWriter(BackendTester): def setUp(self): diff --git a/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py index 64386e5034ec..26d5c474f7a1 100644 --- a/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py +++ b/python/mozbuild/mozbuild/test/backend/test_partialconfigenvironment.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import buildconfig import os import unittest @@ -63,7 +61,7 @@ class TestPartial(unittest.TestCase): myconfig = config.copy() env.write_vars(myconfig) with self.assertRaises(KeyError): - _ = env.substs['MYSUBST'] + x = env.substs['MYSUBST'] self.assertFalse(os.path.exists(path)) myconfig['substs']['MYSUBST'] = 'new' @@ -75,7 +73,7 @@ class TestPartial(unittest.TestCase): del myconfig['substs']['MYSUBST'] env.write_vars(myconfig) with self.assertRaises(KeyError): - _ = env.substs['MYSUBST'] + x = env.substs['MYSUBST'] # Now that the subst is gone, the file still needs to be present so that # make can update dependencies correctly. Overwriting the file with # 'None' is the same as deleting it as far as the @@ -84,8 +82,7 @@ class TestPartial(unittest.TestCase): self.assertTrue(os.path.exists(path)) def _assert_deps(self, env, deps): - deps = sorted(['$(wildcard %s)' % - (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps]) + deps = sorted(['$(wildcard %s)' % (mozpath.join(env.topobjdir, 'config.statusd', d)) for d in deps]) self.assertEqual(sorted(env.get_dependencies()), deps) def test_dependencies(self): @@ -109,9 +106,8 @@ class TestPartial(unittest.TestCase): self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1']) with self.assertRaises(KeyError): - _ = env.substs['NON_EXISTENT'] - self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', - 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT']) + x = env.substs['NON_EXISTENT'] + self._assert_deps(env, ['defines/MOZ_FOO', 'defines/MOZ_BAR', 'substs/MOZ_SUBST_1', 'substs/NON_EXISTENT']) self.assertEqual(env.substs.get('NON_EXISTENT'), None) def test_set_subst(self): @@ -163,6 +159,5 @@ class TestPartial(unittest.TestCase): self.assertEqual(mydefines['DEBUG'], '1') self.assertEqual(mydefines['MOZ_FOO'], '1') - if __name__ == "__main__": main() diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index 5c18a7870200..0d6fd5214fef 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -2,9 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import cPickle as pickle +import json import os import unittest @@ -58,10 +59,9 @@ class TestRecursiveMakeTraversal(unittest.TestCase): traversal.add('X') parallels = set(('G', 'H', 'I', 'J', 'O', 'P', 'Q', 'R', 'U')) - def filter(current, subdirs): return (current, [d for d in subdirs.dirs if d in parallels], - [d for d in subdirs.dirs if d not in parallels]) + [d for d in subdirs.dirs if d not in parallels]) start, deps = traversal.compute_dependencies(filter) self.assertEqual(start, ('X',)) @@ -95,8 +95,8 @@ class TestRecursiveMakeTraversal(unittest.TestCase): self.assertEqual(list(traversal.traverse('', filter)), ['', 'A', 'B', 'E', 'F', 'C', 'G', 'H', 'D', 'I', - 'M', 'N', 'T', 'J', 'O', 'P', 'U', 'K', 'Q', 'R', - 'V', 'L', 'S', 'W', 'X']) + 'M', 'N', 'T', 'J', 'O', 'P', 'U', 'K', 'Q', 'R', + 'V', 'L', 'S', 'W', 'X']) self.assertEqual(list(traversal.traverse('C', filter)), ['C', 'G', 'H']) @@ -191,15 +191,14 @@ class TestRecursiveMakeTraversal(unittest.TestCase): 'J': ('',), }) - class TestRecursiveMakeBackend(BackendTester): def test_basic(self): """Ensure the RecursiveMakeBackend works without error.""" env = self._consume('stub0', RecursiveMakeBackend) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.RecursiveMakeBackend'))) + 'backend.RecursiveMakeBackend'))) self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, - 'backend.RecursiveMakeBackend.in'))) + 'backend.RecursiveMakeBackend.in'))) def test_output_files(self): """Ensure proper files are generated.""" @@ -386,7 +385,7 @@ class TestRecursiveMakeBackend(BackendTester): # EXPORTS files should appear in the dist_include install manifest. m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', 'dist_include')) + '_build_manifests', 'install', 'dist_include')) self.assertEqual(len(m), 7) self.assertIn('foo.h', m) self.assertIn('mozilla/mozilla1.h', m) @@ -407,7 +406,7 @@ class TestRecursiveMakeBackend(BackendTester): 'EXTRA_MDDEPEND_FILES += bar.c.pp', '$(MDDEPDIR)/bar.c.stub: %s/generate-bar.py' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir, '@$(TOUCH) $@', '', 'export:: $(MDDEPDIR)/foo.c.stub', @@ -417,7 +416,7 @@ class TestRecursiveMakeBackend(BackendTester): 'EXTRA_MDDEPEND_FILES += foo.c.pp', '$(MDDEPDIR)/foo.c.stub: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir), '$(REPORT_BUILD)', - '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(MDDEPDIR)/foo.c.stub $(srcdir)/foo-data)' % (env.topsrcdir), # noqa + '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(MDDEPDIR)/foo.c.stub $(srcdir)/foo-data)' % (env.topsrcdir), '@$(TOUCH) $@', '', ] @@ -440,7 +439,7 @@ class TestRecursiveMakeBackend(BackendTester): 'EXTRA_MDDEPEND_FILES += bar.c.pp', '$(MDDEPDIR)/bar.c.stub: %s/generate-bar.py FORCE' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir, '@$(TOUCH) $@', '', 'export:: $(MDDEPDIR)/foo.c.stub', @@ -450,7 +449,7 @@ class TestRecursiveMakeBackend(BackendTester): 'EXTRA_MDDEPEND_FILES += foo.c.pp', '$(MDDEPDIR)/foo.c.stub: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir), '$(REPORT_BUILD)', - '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(MDDEPDIR)/foo.c.stub $(srcdir)/foo-data)' % (env.topsrcdir), # noqa + '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(MDDEPDIR)/foo.c.stub $(srcdir)/foo-data)' % (env.topsrcdir), '@$(TOUCH) $@', '', ] @@ -471,9 +470,9 @@ class TestRecursiveMakeBackend(BackendTester): 'GARBAGE += foo.xyz', 'GARBAGE += $(MDDEPDIR)/foo.xyz.stub', 'EXTRA_MDDEPEND_FILES += foo.xyz.pp', - '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, # noqa + '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, '@$(TOUCH) $@', '', 'LOCALIZED_FILES_0_FILES += foo.xyz', @@ -498,9 +497,9 @@ class TestRecursiveMakeBackend(BackendTester): 'GARBAGE += foo.xyz', 'GARBAGE += $(MDDEPDIR)/foo.xyz.stub', 'EXTRA_MDDEPEND_FILES += foo.xyz.pp', - '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, # noqa + '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, '@$(TOUCH) $@', '', 'libs:: $(MDDEPDIR)/abc.xyz.stub', @@ -508,9 +507,9 @@ class TestRecursiveMakeBackend(BackendTester): 'GARBAGE += abc.xyz', 'GARBAGE += $(MDDEPDIR)/abc.xyz.stub', 'EXTRA_MDDEPEND_FILES += abc.xyz.pp', - '$(MDDEPDIR)/abc.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir, # noqa + '$(MDDEPDIR)/abc.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(MDDEPDIR)/abc.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(MDDEPDIR)/abc.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, '@$(TOUCH) $@', '', ] @@ -532,9 +531,9 @@ class TestRecursiveMakeBackend(BackendTester): 'GARBAGE += foo$(AB_CD).xyz', 'GARBAGE += $(MDDEPDIR)/foo$(AB_CD).xyz.stub', 'EXTRA_MDDEPEND_FILES += foo$(AB_CD).xyz.pp', - '$(MDDEPDIR)/foo$(AB_CD).xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, # noqa + '$(MDDEPDIR)/foo$(AB_CD).xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo$(AB_CD).xyz $(MDDEPDIR)/foo$(AB_CD).xyz.pp $(MDDEPDIR)/foo$(AB_CD).xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo$(AB_CD).xyz $(MDDEPDIR)/foo$(AB_CD).xyz.pp $(MDDEPDIR)/foo$(AB_CD).xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir, '@$(TOUCH) $@', '', 'include $(topsrcdir)/config/AB_rCD.mk', @@ -542,18 +541,18 @@ class TestRecursiveMakeBackend(BackendTester): 'GARBAGE += bar$(AB_rCD).xyz', 'GARBAGE += $(MDDEPDIR)/bar$(AB_rCD).xyz.stub', 'EXTRA_MDDEPEND_FILES += bar$(AB_rCD).xyz.pp', - '$(MDDEPDIR)/bar$(AB_rCD).xyz.stub: %s/generate-foo.py $(call MERGE_RELATIVE_FILE,localized-input,inner/locales) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, # noqa + '$(MDDEPDIR)/bar$(AB_rCD).xyz.stub: %s/generate-foo.py $(call MERGE_RELATIVE_FILE,localized-input,inner/locales) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main bar$(AB_rCD).xyz $(MDDEPDIR)/bar$(AB_rCD).xyz.pp $(MDDEPDIR)/bar$(AB_rCD).xyz.stub $(call MERGE_RELATIVE_FILE,localized-input,inner/locales) $(srcdir)/non-localized-input)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main bar$(AB_rCD).xyz $(MDDEPDIR)/bar$(AB_rCD).xyz.pp $(MDDEPDIR)/bar$(AB_rCD).xyz.stub $(call MERGE_RELATIVE_FILE,localized-input,inner/locales) $(srcdir)/non-localized-input)' % env.topsrcdir, '@$(TOUCH) $@', '', 'zot$(AB_rCD).xyz: $(MDDEPDIR)/zot$(AB_rCD).xyz.stub ;', 'GARBAGE += zot$(AB_rCD).xyz', 'GARBAGE += $(MDDEPDIR)/zot$(AB_rCD).xyz.stub', 'EXTRA_MDDEPEND_FILES += zot$(AB_rCD).xyz.pp', - '$(MDDEPDIR)/zot$(AB_rCD).xyz.stub: %s/generate-foo.py $(call MERGE_RELATIVE_FILE,localized-input,locales) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, # noqa + '$(MDDEPDIR)/zot$(AB_rCD).xyz.stub: %s/generate-foo.py $(call MERGE_RELATIVE_FILE,localized-input,locales) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir, '$(REPORT_BUILD)', - '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main zot$(AB_rCD).xyz $(MDDEPDIR)/zot$(AB_rCD).xyz.pp $(MDDEPDIR)/zot$(AB_rCD).xyz.stub $(call MERGE_RELATIVE_FILE,localized-input,locales) $(srcdir)/non-localized-input)' % env.topsrcdir, # noqa + '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main zot$(AB_rCD).xyz $(MDDEPDIR)/zot$(AB_rCD).xyz.pp $(MDDEPDIR)/zot$(AB_rCD).xyz.stub $(call MERGE_RELATIVE_FILE,localized-input,locales) $(srcdir)/non-localized-input)' % env.topsrcdir, '@$(TOUCH) $@', '', ] @@ -568,7 +567,7 @@ class TestRecursiveMakeBackend(BackendTester): # EXPORTS files should appear in the dist_include install manifest. m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', 'dist_include')) + '_build_manifests', 'install', 'dist_include')) self.assertEqual(len(m), 8) self.assertIn('foo.h', m) self.assertIn('mozilla/mozilla1.h', m) @@ -606,7 +605,7 @@ class TestRecursiveMakeBackend(BackendTester): # RESOURCE_FILES should appear in the dist_bin install manifest. m = InstallManifest(path=os.path.join(env.topobjdir, - '_build_manifests', 'install', 'dist_bin')) + '_build_manifests', 'install', 'dist_bin')) self.assertEqual(len(m), 10) self.assertIn('res/foo.res', m) self.assertIn('res/fonts/font1.ttf', m) @@ -638,7 +637,7 @@ class TestRecursiveMakeBackend(BackendTester): """Pattern matches in test manifests' support-files should be recorded.""" env = self._consume('test-manifests-written', RecursiveMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', '_test_files')) + '_build_manifests', 'install', '_test_files')) # This is not the most robust test in the world, but it gets the job # done. @@ -691,7 +690,7 @@ class TestRecursiveMakeBackend(BackendTester): # Install manifests should contain entries. install_dir = mozpath.join(env.topobjdir, '_build_manifests', - 'install') + 'install') self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'xpidl'))) m = InstallManifest(path=mozpath.join(install_dir, 'xpidl')) @@ -711,7 +710,7 @@ class TestRecursiveMakeBackend(BackendTester): def test_test_support_files_tracked(self): env = self._consume('test-support-binaries-tracked', RecursiveMakeBackend) m = InstallManifest(path=mozpath.join(env.topobjdir, - '_build_manifests', 'install', '_tests')) + '_build_manifests', 'install', '_tests')) self.assertEqual(len(m), 4) self.assertIn('xpcshell/tests/mozbuildtest/test-library.dll', m) self.assertIn('xpcshell/tests/mozbuildtest/test-one.exe', m) @@ -756,8 +755,7 @@ class TestRecursiveMakeBackend(BackendTester): self.assertEqual(m, m2) def test_ipdl_sources(self): - """Test that PREPROCESSED_IPDL_SOURCES and IPDL_SOURCES are written to - ipdlsrcs.mk correctly.""" + """Test that PREPROCESSED_IPDL_SOURCES and IPDL_SOURCES are written to ipdlsrcs.mk correctly.""" env = self._get_environment('ipdl_sources') # Make substs writable so we can set the value of IPDL_ROOT to reflect @@ -774,7 +772,7 @@ class TestRecursiveMakeBackend(BackendTester): topsrcdir = env.topsrcdir.replace(os.sep, '/') expected = [ - "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4), # noqa + "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4), "CPPSRCS := UnifiedProtocols0.cpp", "IPDLDIRS := %s %s/bar %s/foo" % (env.topobjdir, topsrcdir, topsrcdir), ] @@ -819,6 +817,8 @@ class TestRecursiveMakeBackend(BackendTester): backend_path = mozpath.join(env.topobjdir, 'backend.mk') lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + topobjdir = env.topobjdir.replace('\\', '/') + expected = [ 'LOCAL_INCLUDES += -I$(CURDIR)/bar/baz', 'LOCAL_INCLUDES += -I$(CURDIR)/foo', @@ -837,7 +837,7 @@ class TestRecursiveMakeBackend(BackendTester): if not l.startswith('COMPUTED_')] expected = [ - 'RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libtest_library.a' % env.topobjdir, # noqa + 'RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libtest_library.a' % env.topobjdir, 'CARGO_FILE := $(srcdir)/Cargo.toml', 'CARGO_TARGET_DIR := %s' % env.topobjdir, ] @@ -854,7 +854,7 @@ class TestRecursiveMakeBackend(BackendTester): if not l.startswith('COMPUTED_')] expected = [ - 'HOST_RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libhostrusttool.a' % env.topobjdir, # noqa + 'HOST_RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libhostrusttool.a' % env.topobjdir, 'CARGO_FILE := $(srcdir)/Cargo.toml', 'CARGO_TARGET_DIR := %s' % env.topobjdir, ] @@ -871,7 +871,7 @@ class TestRecursiveMakeBackend(BackendTester): if not l.startswith('COMPUTED_')] expected = [ - 'HOST_RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libhostrusttool.a' % env.topobjdir, # noqa + 'HOST_RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libhostrusttool.a' % env.topobjdir, 'CARGO_FILE := $(srcdir)/Cargo.toml', 'CARGO_TARGET_DIR := %s' % env.topobjdir, 'HOST_RUST_LIBRARY_FEATURES := musthave cantlivewithout', @@ -889,7 +889,7 @@ class TestRecursiveMakeBackend(BackendTester): if not l.startswith('COMPUTED_')] expected = [ - 'RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libfeature_library.a' % env.topobjdir, # noqa + 'RUST_LIBRARY_FILE := %s/x86_64-unknown-linux-gnu/release/libfeature_library.a' % env.topobjdir, 'CARGO_FILE := $(srcdir)/Cargo.toml', 'CARGO_TARGET_DIR := %s' % env.topobjdir, 'RUST_LIBRARY_FEATURES := musthave cantlivewithout', @@ -926,7 +926,7 @@ class TestRecursiveMakeBackend(BackendTester): """Test that FINAL_TARGET is written to backend.mk correctly.""" env = self._consume('final_target', RecursiveMakeBackend) - final_target_rule = "FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)" # noqa + final_target_rule = "FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)" expected = dict() expected[env.topobjdir] = [] expected[mozpath.join(env.topobjdir, 'both')] = [ @@ -949,8 +949,8 @@ class TestRecursiveMakeBackend(BackendTester): backend_path = mozpath.join(key, 'backend.mk') lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] found = [str for str in lines if - str.startswith('FINAL_TARGET') or str.startswith('XPI_NAME') or - str.startswith('DIST_SUBDIR')] + str.startswith('FINAL_TARGET') or str.startswith('XPI_NAME') or + str.startswith('DIST_SUBDIR')] self.assertEqual(found, expected_rules) def test_final_target_pp_files(self): @@ -1149,7 +1149,7 @@ class TestRecursiveMakeBackend(BackendTester): def test_test_manifests_duplicate_support_files(self): """Ensure duplicate support-files in test manifests work.""" env = self._consume('test-manifests-duplicate-support-files', - RecursiveMakeBackend) + RecursiveMakeBackend) p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_test_files') m = InstallManifest(p) diff --git a/python/mozbuild/mozbuild/test/backend/test_test_manifest.py b/python/mozbuild/mozbuild/test/backend/test_test_manifest.py index 4574a3df2e94..250eb401d80a 100644 --- a/python/mozbuild/mozbuild/test/backend/test_test_manifest.py +++ b/python/mozbuild/mozbuild/test/backend/test_test_manifest.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import cPickle as pickle import os diff --git a/python/mozbuild/mozbuild/test/backend/test_visualstudio.py b/python/mozbuild/mozbuild/test/backend/test_visualstudio.py index 46d34f6ef875..bfc95e55271a 100644 --- a/python/mozbuild/mozbuild/test/backend/test_visualstudio.py +++ b/python/mozbuild/mozbuild/test/backend/test_visualstudio.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals from xml.dom.minidom import parse import os @@ -41,7 +41,7 @@ class TestVisualStudioBackend(BackendTester): els = d.getElementsByTagName('NMakeForcedIncludes') self.assertEqual(len(els), 1) self.assertEqual(els[0].firstChild.nodeValue, - '$(TopObjDir)\\dist\\include\\mozilla-config.h') + '$(TopObjDir)\\dist\\include\\mozilla-config.h') # LOCAL_INCLUDES get added to the include search path. els = d.getElementsByTagName('NMakeIncludeSearchPath') diff --git a/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py b/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py index 49b5f36e0ac8..b1b72438a6d4 100644 --- a/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py +++ b/python/mozbuild/mozbuild/test/codecoverage/test_lcov_rewrite.py @@ -2,9 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os +import subprocess import unittest import shutil from StringIO import StringIO @@ -109,7 +108,6 @@ LH:2 end_of_record """ - class TempFile(): def __init__(self, content): self.file = NamedTemporaryFile(delete=False) @@ -144,7 +142,6 @@ class TestLcovParser(unittest.TestCase): output = self.parser_roundtrip(fn_with_multiple_commas) self.assertEqual(fn_with_multiple_commas, output) - multiple_included_files = """//@line 1 "/src/dir/foo.js" bazfoobar //@line 2 "/src/dir/path/bar.js" @@ -161,7 +158,6 @@ baz fin """ - class TestLineRemapping(unittest.TestCase): def setUp(self): chrome_map_file = os.path.join(buildconfig.topobjdir, 'chrome-map.json') @@ -241,7 +237,6 @@ class TestLineRemapping(unittest.TestCase): # Rewrite preprocessed entries. lcov_file = lcov_rewriter.LcovFile([fpath]) r_num = [] - def rewrite_source(s): r_num.append(1) return s, pp_remap @@ -267,7 +262,6 @@ class TestLineRemapping(unittest.TestCase): self.assertEqual(original_covered_function_count, sum(r.covered_function_count for r in records)) - class TestUrlFinder(unittest.TestCase): def setUp(self): chrome_map_file = os.path.join(buildconfig.topobjdir, 'chrome-map.json') @@ -288,8 +282,7 @@ class TestUrlFinder(unittest.TestCase): ], }, { - 'chrome://global/content/netError.xhtml': - 'chrome://browser/content/aboutNetError.xhtml', + 'chrome://global/content/netError.xhtml': 'chrome://browser/content/aboutNetError.xhtml', }, { 'dist/bin/components/MainProcessSingleton.js': [ @@ -336,10 +329,8 @@ class TestUrlFinder(unittest.TestCase): omnijar_name = buildconfig.substs.get('OMNIJAR_NAME') paths = [ - ('jar:file:///home/worker/workspace/build/application/' + app_name + - '/' + omnijar_name + '!/components/MainProcessSingleton.js', 'path1'), - ('jar:file:///home/worker/workspace/build/application/' + app_name + - '/browser/features/firefox@getpocket.com.xpi!/bootstrap.js', 'path4'), + ('jar:file:///home/worker/workspace/build/application/' + app_name + '/' + omnijar_name + '!/components/MainProcessSingleton.js', 'path1'), + ('jar:file:///home/worker/workspace/build/application/' + app_name + '/browser/features/firefox@getpocket.com.xpi!/bootstrap.js', 'path4'), ] url_finder = lcov_rewriter.UrlFinder(self._chrome_map_file, '', '', []) @@ -347,6 +338,9 @@ class TestUrlFinder(unittest.TestCase): self.assertEqual(url_finder.rewrite_url(path)[0], expected) def test_wrong_scheme_paths(self): + app_name = buildconfig.substs.get('MOZ_APP_NAME') + omnijar_name = buildconfig.substs.get('OMNIJAR_NAME') + paths = [ 'http://www.mozilla.org/aFile.js', 'https://www.mozilla.org/aFile.js', @@ -362,37 +356,31 @@ class TestUrlFinder(unittest.TestCase): def test_chrome_resource_paths(self): paths = [ # Path with default url prefix - ('resource://gre/modules/osfile/osfile_async_worker.js', - ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), + ('resource://gre/modules/osfile/osfile_async_worker.js', ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), # Path with url prefix that is in chrome map - ('resource://activity-stream/lib/PrefsFeed.jsm', - ('browser/components/newtab/lib/PrefsFeed.jsm', None)), + ('resource://activity-stream/lib/PrefsFeed.jsm', ('browser/components/newtab/lib/PrefsFeed.jsm', None)), # Path which is in url overrides - ('chrome://global/content/netError.xhtml', - ('browser/base/content/aboutNetError.xhtml', None)), + ('chrome://global/content/netError.xhtml', ('browser/base/content/aboutNetError.xhtml', None)), # Path which ends with > eval ('resource://gre/modules/osfile/osfile_async_worker.js line 3 > eval', None), # Path which ends with > Function ('resource://gre/modules/osfile/osfile_async_worker.js line 3 > Function', None), # Path which contains "->" - ('resource://gre/modules/addons/XPIProvider.jsm -> resource://gre/modules/osfile/osfile_async_worker.js', # noqa - ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), + ('resource://gre/modules/addons/XPIProvider.jsm -> resource://gre/modules/osfile/osfile_async_worker.js', ('toolkit/components/osfile/modules/osfile_async_worker.js', None)), # Path with pp_info ('resource://gre/modules/AppConstants.jsm', ('toolkit/modules/AppConstants.jsm', { '101,102': [ - 'toolkit/modules/AppConstants.jsm', + 'toolkit/modules/AppConstants.jsm', 135 ], })), # Path with query - ('resource://activity-stream/lib/PrefsFeed.jsm?q=0.9098419174803978', - ('browser/components/newtab/lib/PrefsFeed.jsm', None)), + ('resource://activity-stream/lib/PrefsFeed.jsm?q=0.9098419174803978', ('browser/components/newtab/lib/PrefsFeed.jsm', None)), ] url_finder = lcov_rewriter.UrlFinder(self._chrome_map_file, '', 'dist/bin/', []) for path, expected in paths: self.assertEqual(url_finder.rewrite_url(path), expected) - if __name__ == '__main__': mozunit.main() diff --git a/python/mozbuild/mozbuild/test/common.py b/python/mozbuild/mozbuild/test/common.py index 0ed5ff7c523d..3b258d2af1f5 100644 --- a/python/mozbuild/mozbuild/test/common.py +++ b/python/mozbuild/mozbuild/test/common.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import errno import os @@ -46,7 +46,7 @@ class MockConfig(object): topsrcdir='/path/to/topsrcdir', extra_substs={}, error_is_fatal=True, - ): + ): self.topsrcdir = mozpath.abspath(topsrcdir) self.topobjdir = mozpath.abspath('/path/to/topobjdir') diff --git a/python/mozbuild/mozbuild/test/compilation/test_warnings.py b/python/mozbuild/mozbuild/test/compilation/test_warnings.py index 615b187c7800..1aa1fd1d6f9d 100644 --- a/python/mozbuild/mozbuild/test/compilation/test_warnings.py +++ b/python/mozbuild/mozbuild/test/compilation/test_warnings.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import unittest @@ -29,12 +27,11 @@ MSVC_TESTS = [ "conversion from 'double' to 'uint32_t', possible loss of data", 'C:/mozilla-central/test/foo.cpp', 793, 'C4244', "'return' : conversion from 'double' to 'uint32_t', possible loss of " - 'data') + 'data') ] CURRENT_LINE = 1 - def get_warning(): global CURRENT_LINE @@ -48,7 +45,6 @@ def get_warning(): return w - class TestCompilerWarning(unittest.TestCase): def test_equivalence(self): w1 = CompilerWarning() @@ -126,7 +122,6 @@ class TestCompilerWarning(unittest.TestCase): self.assertGreaterEqual(w2, w1) self.assertGreaterEqual(w1, w2) - class TestWarningsParsing(unittest.TestCase): def test_clang_parsing(self): for source, filename, line, column, message, flag in CLANG_TESTS: @@ -153,7 +148,6 @@ class TestWarningsParsing(unittest.TestCase): self.assertEqual(warning['flag'], flag) self.assertEqual(warning['message'], message) - class TestWarningsDatabase(unittest.TestCase): def test_basic(self): db = WarningsDatabase() diff --git a/python/mozbuild/mozbuild/test/configure/common.py b/python/mozbuild/mozbuild/test/configure/common.py index 97760b0b958a..91682fbd48aa 100644 --- a/python/mozbuild/mozbuild/test/configure/common.py +++ b/python/mozbuild/mozbuild/test/configure/common.py @@ -31,7 +31,6 @@ def fake_short_path(path): for p in mozpath.split(path)) return path - def ensure_exe_extension(path): if sys.platform.startswith('win'): return path + '.exe' @@ -74,7 +73,6 @@ class ConfigureTestSandbox(ConfigureSandbox): This class is only meant to implement the minimal things to make moz.configure testing possible. As such, it takes shortcuts. ''' - def __init__(self, paths, config, environ, *args, **kwargs): self._search_path = environ.get('PATH', '').split(os.pathsep) @@ -155,6 +153,7 @@ class ConfigureTestSandbox(ConfigureSandbox): def __call__(self, *args, **kwargs): return self._func(*args, **kwargs) + return ReadOnlyNamespace( create_unicode_buffer=self.create_unicode_buffer, windll=ReadOnlyNamespace( diff --git a/python/mozbuild/mozbuild/test/configure/lint.py b/python/mozbuild/mozbuild/test/configure/lint.py index ee1394b169f4..9bc5add09c31 100644 --- a/python/mozbuild/mozbuild/test/configure/lint.py +++ b/python/mozbuild/mozbuild/test/configure/lint.py @@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals import os import unittest +from StringIO import StringIO from mozunit import main from buildconfig import ( topobjdir, diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py index 2e09ca8880df..671c9582d496 100644 --- a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -233,6 +233,7 @@ class TestChecksConfigure(unittest.TestCase): self.assertEqual(config, {'FOO': self.KNOWN_A}) self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + def test_check_prog_with_args(self): config, out, status = self.get_result( 'check_prog("FOO", ("unknown", "known-b", "known c"))', @@ -430,8 +431,7 @@ class TestChecksConfigure(unittest.TestCase): 'single element, or a string') def test_check_prog_with_path(self): - config, out, status = self.get_result( - 'check_prog("A", ("known-a",), paths=["/some/path"])') + config, out, status = self.get_result('check_prog("A", ("known-a",), paths=["/some/path"])') self.assertEqual(status, 1) self.assertEqual(config, {}) self.assertEqual(out, textwrap.dedent('''\ @@ -474,8 +474,7 @@ class TestChecksConfigure(unittest.TestCase): self.assertEqual(status, 1) self.assertEqual(config, {}) self.assertEqual(out, textwrap.dedent('''\ - checking for a... ''' # noqa # trailing whitespace... - ''' + checking for a... DEBUG: a: Trying known-a ERROR: Paths provided to find_program must be a list of strings, not %r ''' % mozpath.dirname(self.OTHER_A))) @@ -621,11 +620,8 @@ class TestChecksConfigure(unittest.TestCase): self.assertEqual(out, textwrap.dedent('''\ checking for java... %s checking for jarsigner... not found - ERROR: The program jarsigner was not found. Set $JAVA_HOME to your \ -Java SDK directory or use '--with-java-bin-path={java-bin-dir}' - ''' % (java) - ), - ) + ERROR: The program jarsigner was not found. Set $JAVA_HOME to your Java SDK directory or use '--with-java-bin-path={java-bin-dir}' + ''' % (java))) def test_pkg_check_modules(self): mock_pkg_config_version = '0.10.0' @@ -668,6 +664,7 @@ Java SDK directory or use '--with-java-bin-path={java-bin-dir}' extra_paths = { mock_pkg_config_path: mock_pkg_config, } + includes = ('util.configure', 'checks.configure', 'pkg.configure') config, output, status = get_result("pkg_check_modules('MOZ_VALID', 'valid')") self.assertEqual(status, 1) @@ -678,6 +675,7 @@ Java SDK directory or use '--with-java-bin-path={java-bin-dir}' *** to the full path to pkg-config. ''')) + config, output, status = get_result("pkg_check_modules('MOZ_VALID', 'valid')", extra_paths=extra_paths) self.assertEqual(status, 0) diff --git a/python/mozbuild/mozbuild/test/configure/test_compile_checks.py b/python/mozbuild/mozbuild/test/configure/test_compile_checks.py index ce2c7938d26a..618a30bdd868 100644 --- a/python/mozbuild/mozbuild/test/configure/test_compile_checks.py +++ b/python/mozbuild/mozbuild/test/configure/test_compile_checks.py @@ -21,7 +21,6 @@ from test_toolchain_helpers import FakeCompiler class BaseCompileChecks(unittest.TestCase): def get_mock_compiler(self, expected_test_content=None, expected_flags=None): expected_flags = expected_flags or [] - def mock_compiler(stdin, args): args, test_file = args[:-1], args[-1] self.assertIn('-c', args) @@ -221,7 +220,7 @@ class TestHeaderChecks(BaseCompileChecks): config, out, status = self.do_compile_test(cmd) self.assertEqual(status, 0) self.assertEqual(out, '') - self.assertEqual(config, {'DEFINES': {}}) + self.assertEqual(config, {'DEFINES':{}}) def test_check_header_include(self): expected_test_content = textwrap.dedent('''\ diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py index 4f87df99625b..3ae4bbdd8245 100644 --- a/python/mozbuild/mozbuild/test/configure/test_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_configure.py @@ -247,16 +247,16 @@ class TestConfigure(unittest.TestCase): def foo(): import sys foo()'''), - sandbox - ) + sandbox + ) exec_(textwrap.dedent(''' @template @imports('sys') def foo(): return sys'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sys) @@ -265,8 +265,8 @@ class TestConfigure(unittest.TestCase): @imports(_from='os', _import='path') def foo(): return path'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), os.path) @@ -275,8 +275,8 @@ class TestConfigure(unittest.TestCase): @imports(_from='os', _import='path', _as='os_path') def foo(): return os_path'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), os.path) @@ -285,8 +285,8 @@ class TestConfigure(unittest.TestCase): @imports('__builtin__') def foo(): return __builtin__'''), - sandbox - ) + sandbox + ) import __builtin__ self.assertIs(sandbox['foo'](), __builtin__) @@ -296,8 +296,8 @@ class TestConfigure(unittest.TestCase): @imports(_from='__builtin__', _import='open') def foo(): return open('%s')''' % os.devnull), - sandbox - ) + sandbox + ) f = sandbox['foo']() self.assertEquals(f.name, os.devnull) @@ -310,8 +310,8 @@ class TestConfigure(unittest.TestCase): def foo(): import sys return sys'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sys) @@ -320,8 +320,8 @@ class TestConfigure(unittest.TestCase): @imports('__sandbox__') def foo(): return __sandbox__'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sandbox) @@ -330,8 +330,8 @@ class TestConfigure(unittest.TestCase): @imports(_import='__sandbox__', _as='s') def foo(): return s'''), - sandbox - ) + sandbox + ) self.assertIs(sandbox['foo'](), sandbox) @@ -348,8 +348,8 @@ class TestConfigure(unittest.TestCase): return sys return bar bar = foo()'''), - sandbox - ) + sandbox + ) with self.assertRaises(NameError) as e: sandbox._depends[sandbox['bar']].result() @@ -377,8 +377,8 @@ class TestConfigure(unittest.TestCase): return sys foo() foo()'''), - sandbox - ) + sandbox + ) self.assertEquals(len(imports), 1) @@ -587,7 +587,7 @@ class TestConfigure(unittest.TestCase): config = get_config(['--enable-foo=a,b']) self.assertIn('BAR', config) - self.assertEquals(config['BAR'], PositiveOptionValue(('a', 'b'))) + self.assertEquals(config['BAR'], PositiveOptionValue(('a','b'))) with self.assertRaises(InvalidOptionError) as e: get_config(['--enable-foo=a,b', '--disable-bar']) @@ -639,20 +639,18 @@ class TestConfigure(unittest.TestCase): mozpath.join(test_data_path, 'imply_option', 'imm.configure')) with self.assertRaisesRegexp(InvalidOptionError, - "--enable-foo' implied by 'imply_option at %s:7' conflicts " - "with '--disable-foo' from the command-line" % config_path): + "--enable-foo' implied by 'imply_option at %s:7' conflicts with " + "'--disable-foo' from the command-line" % config_path): get_config(['--disable-foo']) with self.assertRaisesRegexp(InvalidOptionError, - "--enable-bar=foo,bar' implied by 'imply_option at %s:16' " - "conflicts with '--enable-bar=a,b,c' from the command-line" - % config_path): + "--enable-bar=foo,bar' implied by 'imply_option at %s:16' conflicts" + " with '--enable-bar=a,b,c' from the command-line" % config_path): get_config(['--enable-bar=a,b,c']) with self.assertRaisesRegexp(InvalidOptionError, - "--enable-baz=BAZ' implied by 'imply_option at %s:25' " - "conflicts with '--enable-baz=QUUX' from the command-line" - % config_path): + "--enable-baz=BAZ' implied by 'imply_option at %s:25' conflicts" + " with '--enable-baz=QUUX' from the command-line" % config_path): get_config(['--enable-baz=QUUX']) def test_imply_option_failures(self): @@ -812,17 +810,17 @@ class TestConfigure(unittest.TestCase): # imply_options resolve to None, which disables the imply_option. with self.assertRaises(ConfigureError) as e: - self.get_config() + config = self.get_config() self.assertEquals(e.exception.message, message) with self.assertRaises(ConfigureError) as e: - self.get_config(['--with-qux']) + config = self.get_config(['--with-qux']) self.assertEquals(e.exception.message, message) with self.assertRaises(ConfigureError) as e: - self.get_config(['--without-foo', '--with-qux']) + config = self.get_config(['--without-foo', '--with-qux']) self.assertEquals(e.exception.message, message) diff --git a/python/mozbuild/mozbuild/test/configure/test_lint.py b/python/mozbuild/mozbuild/test/configure/test_lint.py index b467d0c3d3bd..9d7dd425b46d 100644 --- a/python/mozbuild/mozbuild/test/configure/test_lint.py +++ b/python/mozbuild/mozbuild/test/configure/test_lint.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function, unicode_literals +from StringIO import StringIO import contextlib import os import sys diff --git a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py index 6828ed06c8e5..fa88e700d0cf 100644 --- a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py @@ -4,6 +4,8 @@ from __future__ import absolute_import, print_function, unicode_literals +import sys + from mozunit import main from mozbuild.util import ( exec_, @@ -154,10 +156,10 @@ class TestMozConfigure(BaseConfigureTest): {'PATH': '/usr/bin', 'MAKENSISU': '/usr/bin/makensis'}) return sandbox._value_for(sandbox['nsis_version']) - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit) as e: check_nsis_version('v2.5') - with self.assertRaises(SystemExit): + with self.assertRaises(SystemExit) as e: check_nsis_version('v3.0a2') self.assertEquals(check_nsis_version('v3.0b1'), '3.0b1') diff --git a/python/mozbuild/mozbuild/test/configure/test_options.py b/python/mozbuild/mozbuild/test/configure/test_options.py index 68f4e884961e..9defccb2c5de 100644 --- a/python/mozbuild/mozbuild/test/configure/test_options.py +++ b/python/mozbuild/mozbuild/test/configure/test_options.py @@ -250,7 +250,7 @@ class TestOption(unittest.TestCase): self.assertEquals(PositiveOptionValue(('c',)), value) value = option.get_value('--with-option=-b,+d') - self.assertEquals(PositiveOptionValue(('c', 'd')), value) + self.assertEquals(PositiveOptionValue(('c','d')), value) # Adding something that is in the default is fine value = option.get_value('--with-option=+b') @@ -875,18 +875,14 @@ class TestCommandLineHelper(unittest.TestCase): bar = Option('--bar', possible_origins=('mozconfig',)) - with self.assertRaisesRegexp( - InvalidOptionError, - "--bar can not be set by command-line. Values are accepted from: mozconfig" - ): + with self.assertRaisesRegexp(InvalidOptionError, + "--bar can not be set by command-line. Values are accepted from: mozconfig"): helper.handle(bar) baz = Option(env='BAZ', possible_origins=('implied',)) - with self.assertRaisesRegexp( - InvalidOptionError, - "BAZ=1 can not be set by environment. Values are accepted from: implied" - ): + with self.assertRaisesRegexp(InvalidOptionError, + "BAZ=1 can not be set by environment. Values are accepted from: implied"): helper.handle(baz) diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py index edfcd6cbc60b..5bdbdcfd2e0a 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py @@ -42,7 +42,6 @@ class CompilerPreprocessor(Preprocessor): # different handling than what our Preprocessor does out of the box. # Hack around it enough that the configure tests work properly. context = self.context - def normalize_numbers(value): if isinstance(value, types.StringTypes): if value[-1:] == 'L' and value[:-1].isdigit(): @@ -50,7 +49,6 @@ class CompilerPreprocessor(Preprocessor): return value # Our Preprocessor doesn't handle macros with parameters, so we hack # around that for __has_feature()-like things. - def normalize_has_feature_or_builtin(expr): return self.HAS_FEATURE_OR_BUILTIN.sub(r'\1\2', expr) self.context = self.Context( @@ -164,7 +162,6 @@ class FakeCompiler(dict): For convenience, FakeCompiler instances can be added (+) to one another. ''' - def __init__(self, *definitions): for definition in definitions: if all(not isinstance(d, dict) for d in definition.itervalues()): diff --git a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py index c8fe93e99c66..2233cff559aa 100644 --- a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py +++ b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py @@ -82,7 +82,7 @@ class TestToolkitMozConfigure(BaseConfigureTest): self.assertEqual(get_value(environ={'MOZILLA_OFFICIAL': 1}), None) self.assertEqual(get_value(['--enable-release'], - environ={'MOZILLA_OFFICIAL': 1}), None) + environ={'MOZILLA_OFFICIAL': 1}), None) with self.assertRaises(InvalidOptionError): get_value(['--disable-release'], @@ -120,8 +120,7 @@ class TestToolkitMozConfigure(BaseConfigureTest): self.assertEqual( out.getvalue(), - ('ERROR: Yasm is required to build with vpx, but you do not appear ' - 'to have Yasm installed.\n'), + 'ERROR: Yasm is required to build with vpx, but you do not appear to have Yasm installed.\n' ) out.truncate(0) @@ -130,8 +129,7 @@ class TestToolkitMozConfigure(BaseConfigureTest): self.assertEqual( out.getvalue(), - ('ERROR: Yasm is required to build with jpeg and vpx, but you do not appear ' - 'to have Yasm installed.\n'), + 'ERROR: Yasm is required to build with jpeg and vpx, but you do not appear to have Yasm installed.\n' ) out.truncate(0) @@ -140,8 +138,7 @@ class TestToolkitMozConfigure(BaseConfigureTest): self.assertEqual( out.getvalue(), - ('ERROR: Yasm is required to build with jpeg, libav and vpx, but you do not appear ' - 'to have Yasm installed.\n'), + 'ERROR: Yasm is required to build with jpeg, libav and vpx, but you do not appear to have Yasm installed.\n' ) out.truncate(0) diff --git a/python/mozbuild/mozbuild/test/configure/test_util.py b/python/mozbuild/mozbuild/test/configure/test_util.py index 122a405c9316..9f9575fd0606 100644 --- a/python/mozbuild/mozbuild/test/configure/test_util.py +++ b/python/mozbuild/mozbuild/test/configure/test_util.py @@ -57,7 +57,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_format' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, err) + handler = ConfigureOutputHandler(out, err) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -79,7 +79,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_continuation' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, out) + handler = ConfigureOutputHandler(out, out) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -138,7 +138,7 @@ class TestConfigureOutputHandler(unittest.TestCase): err = StringIO() logger.removeHandler(handler) - handler = ConfigureOutputHandler(out, err) + handler = ConfigureOutputHandler(out, err) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -167,7 +167,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_queue_debug' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, out, maxlen=3) + handler = ConfigureOutputHandler(out, out, maxlen=3) handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) logger.addHandler(handler) @@ -268,7 +268,7 @@ class TestConfigureOutputHandler(unittest.TestCase): name = '%s.test_queue_debug_reentrant' % self.__class__.__name__ logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) - handler = ConfigureOutputHandler(out, out, maxlen=10) + handler = ConfigureOutputHandler(out, out, maxlen=10) handler.setFormatter(logging.Formatter('%(levelname)s| %(message)s')) logger.addHandler(handler) @@ -424,7 +424,7 @@ class TestLogSubprocessOutput(unittest.TestCase): sandbox = ConfigureSandbox({}, {}, ['configure'], out, out) sandbox.include_file(mozpath.join(topsrcdir, 'build', - 'moz.configure', 'util.configure')) + 'moz.configure', 'util.configure')) sandbox.include_file(mozpath.join(topsrcdir, 'python', 'mozbuild', 'mozbuild', 'test', 'configure', 'data', 'subprocess.configure')) @@ -474,7 +474,6 @@ class TestVersion(unittest.TestCase): self.assertEqual(v.minor, 0) self.assertEqual(v.patch, 0) - class TestCheckCmdOutput(unittest.TestCase): def get_result(self, command='', paths=None): @@ -484,7 +483,7 @@ class TestCheckCmdOutput(unittest.TestCase): sandbox = ConfigureTestSandbox(paths, config, {}, ['/bin/configure'], out, out) sandbox.include_file(mozpath.join(topsrcdir, 'build', - 'moz.configure', 'util.configure')) + 'moz.configure', 'util.configure')) status = 0 try: exec_(command, sandbox) diff --git a/python/mozbuild/mozbuild/test/controller/test_ccachestats.py b/python/mozbuild/mozbuild/test/controller/test_ccachestats.py index ef452e0dea06..60d99c82d5cd 100644 --- a/python/mozbuild/mozbuild/test/controller/test_ccachestats.py +++ b/python/mozbuild/mozbuild/test/controller/test_ccachestats.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import time import unittest @@ -291,6 +291,5 @@ class TestCcacheStats(unittest.TestCase): stat9 = CCacheStats(self.STAT9) self.assertTrue(stat9) - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/controller/test_clobber.py b/python/mozbuild/mozbuild/test/controller/test_clobber.py index 9bc1fcb9d3c7..5aa080337c57 100644 --- a/python/mozbuild/mozbuild/test/controller/test_clobber.py +++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import shutil @@ -123,7 +123,7 @@ class TestClobberer(unittest.TestCase): self.assertFalse(os.path.exists(dummy_path)) self.assertTrue(os.path.exists(c.obj_clobber)) self.assertGreaterEqual(os.path.getmtime(c.obj_clobber), - os.path.getmtime(c.src_clobber)) + os.path.getmtime(c.src_clobber)) def test_objdir_is_srcdir(self): """If topobjdir is the topsrcdir, refuse to clobber.""" @@ -181,6 +181,7 @@ class TestClobberer(unittest.TestCase): self.assertFalse(performed) self.assertIn('Cannot clobber while the shell is inside', reason) + def test_mozconfig_opt_in(self): """Auto clobber iff AUTOCLOBBER is in the environment.""" diff --git a/python/mozbuild/mozbuild/test/frontend/test_context.py b/python/mozbuild/mozbuild/test/frontend/test_context.py index ef0f8fca774b..dcb1b65bdabb 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_context.py +++ b/python/mozbuild/mozbuild/test/frontend/test_context.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import os import unittest @@ -42,20 +40,20 @@ class TestContext(unittest.TestCase): self.assertEqual(test['foo'], 0) - self.assertEqual(set(test.keys()), {'foo'}) + self.assertEqual(set(test.keys()), { 'foo' }) self.assertEqual(test['bar'], False) - self.assertEqual(set(test.keys()), {'foo', 'bar'}) + self.assertEqual(set(test.keys()), { 'foo', 'bar' }) self.assertEqual(test['baz'], {}) - self.assertEqual(set(test.keys()), {'foo', 'bar', 'baz'}) + self.assertEqual(set(test.keys()), { 'foo', 'bar', 'baz' }) with self.assertRaises(KeyError): test['qux'] - self.assertEqual(set(test.keys()), {'foo', 'bar', 'baz'}) + self.assertEqual(set(test.keys()), { 'foo', 'bar', 'baz' }) def test_type_check(self): test = Context({ @@ -77,7 +75,7 @@ class TestContext(unittest.TestCase): test['baz'] = [('a', 1), ('b', 2)] - self.assertEqual(test['baz'], {'a': 1, 'b': 2}) + self.assertEqual(test['baz'], { 'a': 1, 'b': 2 }) def test_update(self): test = Context({ @@ -95,7 +93,7 @@ class TestContext(unittest.TestCase): test.update(bar=True, foo=1) - self.assertEqual(set(test.keys()), {'foo', 'bar'}) + self.assertEqual(set(test.keys()), { 'foo', 'bar' }) self.assertEqual(test['foo'], 1) self.assertEqual(test['bar'], True) @@ -103,13 +101,13 @@ class TestContext(unittest.TestCase): self.assertEqual(test['foo'], 2) self.assertEqual(test['bar'], False) - test.update([('foo', 0), ('baz', {'a': 1, 'b': 2})]) + test.update([('foo', 0), ('baz', { 'a': 1, 'b': 2 })]) self.assertEqual(test['foo'], 0) - self.assertEqual(test['baz'], {'a': 1, 'b': 2}) + self.assertEqual(test['baz'], { 'a': 1, 'b': 2 }) test.update([('foo', 42), ('baz', [('c', 3), ('d', 4)])]) self.assertEqual(test['foo'], 42) - self.assertEqual(test['baz'], {'c': 3, 'd': 4}) + self.assertEqual(test['baz'], { 'c': 3, 'd': 4 }) def test_context_paths(self): test = Context() @@ -216,8 +214,7 @@ class TestContext(unittest.TestCase): self.assertEqual(test.source_stack, [foo, bar, bar, foo]) def test_context_dirs(self): - class Config(object): - pass + class Config(object): pass config = Config() config.topsrcdir = mozpath.abspath(os.curdir) config.topobjdir = mozpath.abspath('obj') @@ -280,8 +277,7 @@ class TestSymbols(unittest.TestCase): class TestPaths(unittest.TestCase): @classmethod def setUpClass(cls): - class Config(object): - pass + class Config(object): pass cls.config = config = Config() config.topsrcdir = mozpath.abspath(os.curdir) config.topobjdir = mozpath.abspath('obj') @@ -727,7 +723,7 @@ class TestFiles(unittest.TestCase): f2 = Files(c, 'b/**', 'a/bar') f2['BUG_COMPONENT'] = (u'Product2', u'Component2') - files = {'a/foo': f1, 'a/bar': f2, 'b/foo': f2} + files = {'a/foo': f1, 'a/bar' : f2, 'b/foo' : f2 } self.assertEqual(Files.aggregate(files), { 'bug_component_counts': [ ((u'Product2', u'Component2'), 2), diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index c93549f52514..5e5ab7ff28b7 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import unittest @@ -14,6 +14,7 @@ from mozbuild.frontend.context import ( Path, ) from mozbuild.frontend.data import ( + ChromeManifestEntry, ComputedFlags, ConfigFileSubstitution, Defines, @@ -22,6 +23,7 @@ from mozbuild.frontend.data import ( FinalTargetPreprocessedFiles, GeneratedFile, GeneratedSources, + HostDefines, HostProgram, HostRustLibrary, HostRustProgram, @@ -50,6 +52,7 @@ from mozbuild.frontend.reader import ( BuildReaderError, SandboxValidationError, ) +from mozpack.chrome import manifest from mozbuild.test.common import MockConfig @@ -171,9 +174,9 @@ class TestEmitterBasic(unittest.TestCase): topobjdir = mozpath.abspath(reader.config.topobjdir) self.assertEqual(objs[0].relpath, 'foo') self.assertEqual(mozpath.normpath(objs[0].output_path), - mozpath.normpath(mozpath.join(topobjdir, 'foo'))) + mozpath.normpath(mozpath.join(topobjdir, 'foo'))) self.assertEqual(mozpath.normpath(objs[1].output_path), - mozpath.normpath(mozpath.join(topobjdir, 'bar'))) + mozpath.normpath(mozpath.join(topobjdir, 'bar'))) def test_variable_passthru(self): reader = self.reader('variable-passthru') @@ -400,7 +403,7 @@ class TestEmitterBasic(unittest.TestCase): def test_resolved_flags_error(self): reader = self.reader('resolved-flags-error') with self.assertRaisesRegexp(BuildReaderError, - "`DEFINES` may not be set in COMPILE_FLAGS from moz.build"): + "`DEFINES` may not be set in COMPILE_FLAGS from moz.build"): self.read_topsrcdir(reader) def test_includes_in_flags(self): @@ -433,7 +436,7 @@ class TestEmitterBasic(unittest.TestCase): # When yasm is not available, this should raise. reader = self.reader('use-yasm') with self.assertRaisesRegexp(SandboxValidationError, - 'yasm is not available'): + 'yasm is not available'): self.read_topsrcdir(reader) # When yasm is available, this should work. @@ -530,22 +533,19 @@ class TestEmitterBasic(unittest.TestCase): LOCALIZED_FILES as an objdir path produces an error. """ reader = self.reader('localized-files-not-localized-generated') - with self.assertRaisesRegexp( - SandboxValidationError, - 'Objdir file listed in LOCALIZED_FILES not in LOCALIZED_GENERATED_FILES:' - ): - self.read_topsrcdir(reader) + with self.assertRaisesRegexp(SandboxValidationError, + 'Objdir file listed in LOCALIZED_FILES not in LOCALIZED_GENERATED_FILES:'): + objs = self.read_topsrcdir(reader) + def test_localized_generated_files_final_target_files(self): """Test that using LOCALIZED_GENERATED_FILES and then putting the output in FINAL_TARGET_FILES as an objdir path produces an error. """ reader = self.reader('localized-generated-files-final-target-files') - with self.assertRaisesRegexp( - SandboxValidationError, - 'Outputs of LOCALIZED_GENERATED_FILES cannot be used in FINAL_TARGET_FILES:' - ): - self.read_topsrcdir(reader) + with self.assertRaisesRegexp(SandboxValidationError, + 'Outputs of LOCALIZED_GENERATED_FILES cannot be used in FINAL_TARGET_FILES:'): + objs = self.read_topsrcdir(reader) def test_generated_files_method_names(self): reader = self.reader('generated-files-method-names') @@ -578,19 +578,19 @@ class TestEmitterBasic(unittest.TestCase): def test_generated_files_no_script(self): reader = self.reader('generated-files-no-script') with self.assertRaisesRegexp(SandboxValidationError, - 'Script for generating bar.c does not exist'): + 'Script for generating bar.c does not exist'): self.read_topsrcdir(reader) def test_generated_files_no_inputs(self): reader = self.reader('generated-files-no-inputs') with self.assertRaisesRegexp(SandboxValidationError, - 'Input for generating foo.c does not exist'): + 'Input for generating foo.c does not exist'): self.read_topsrcdir(reader) def test_generated_files_no_python_script(self): reader = self.reader('generated-files-no-python-script') with self.assertRaisesRegexp(SandboxValidationError, - 'Script for generating bar.c does not end in .py'): + 'Script for generating bar.c does not end in .py'): self.read_topsrcdir(reader) def test_exports(self): @@ -619,7 +619,7 @@ class TestEmitterBasic(unittest.TestCase): ''' reader = self.reader('exports-missing') with self.assertRaisesRegexp(SandboxValidationError, - 'File listed in EXPORTS does not exist:'): + 'File listed in EXPORTS does not exist:'): self.read_topsrcdir(reader) def test_exports_missing_generated(self): @@ -628,7 +628,7 @@ class TestEmitterBasic(unittest.TestCase): ''' reader = self.reader('exports-missing-generated') with self.assertRaisesRegexp(SandboxValidationError, - 'Objdir file listed in EXPORTS not in GENERATED_FILES:'): + 'Objdir file listed in EXPORTS not in GENERATED_FILES:'): self.read_topsrcdir(reader) def test_exports_generated(self): @@ -665,7 +665,7 @@ class TestEmitterBasic(unittest.TestCase): def test_test_harness_files_root(self): reader = self.reader('test-harness-files-root') with self.assertRaisesRegexp(SandboxValidationError, - 'Cannot install files to the root of TEST_HARNESS_FILES'): + 'Cannot install files to the root of TEST_HARNESS_FILES'): self.read_topsrcdir(reader) def test_program(self): @@ -736,6 +736,7 @@ class TestEmitterBasic(unittest.TestCase): with self.assertRaisesRegexp(SandboxValidationError, 'Empty test manifest'): self.read_topsrcdir(reader) + def test_test_manifest_just_support_files(self): """A test manifest with no tests but support-files is not supported.""" reader = self.reader('test-manifest-just-support') @@ -749,11 +750,8 @@ class TestEmitterBasic(unittest.TestCase): """ reader = self.reader('test-manifest-dupes') - with self.assertRaisesRegexp( - SandboxValidationError, - 'bar.js appears multiple times ' - 'in a test manifest under a support-files field, please omit the duplicate entry.' - ): + with self.assertRaisesRegexp(SandboxValidationError, 'bar.js appears multiple times ' + 'in a test manifest under a support-files field, please omit the duplicate entry.'): self.read_topsrcdir(reader) def test_test_manifest_absolute_support_files(self): @@ -932,7 +930,7 @@ class TestEmitterBasic(unittest.TestCase): self.assertEqual(o.dupe_manifest, m.get('dupe', False)) external_normalized = set(mozpath.basename(p) for p in - o.external_installs) + o.external_installs) self.assertEqual(external_normalized, m.get('external', set())) self.assertEqual(len(o.installs), len(m['installs'])) @@ -950,7 +948,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('test-manifest-unmatched-generated') with self.assertRaisesRegexp(SandboxValidationError, - 'entry in generated-files not present elsewhere'): + 'entry in generated-files not present elsewhere'): self.read_topsrcdir(reader), def test_test_manifest_parent_support_files_dir(self): @@ -967,14 +965,14 @@ class TestEmitterBasic(unittest.TestCase): expected = mozpath.join(o.srcdir, 'support-file.txt') self.assertIn(expected, o.installs) self.assertEqual(o.installs[expected], - ('testing/mochitest/tests/child/support-file.txt', False)) + ('testing/mochitest/tests/child/support-file.txt', False)) def test_test_manifest_missing_test_error(self): """Missing test files should result in error.""" reader = self.reader('test-manifest-missing-test-file') with self.assertRaisesRegexp(SandboxValidationError, - 'lists test that does not exist: test_missing.html'): + 'lists test that does not exist: test_missing.html'): self.read_topsrcdir(reader) def test_test_manifest_missing_test_error_unfiltered(self): @@ -982,7 +980,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('test-manifest-missing-test-file-unfiltered') with self.assertRaisesRegexp(SandboxValidationError, - 'lists test that does not exist: missing.js'): + 'lists test that does not exist: missing.js'): self.read_topsrcdir(reader) def test_ipdl_sources(self): @@ -1030,6 +1028,7 @@ class TestEmitterBasic(unittest.TestCase): ]) self.assertEqual(generated_sources, expected) + def test_local_includes(self): """Test that LOCAL_INCLUDES is emitted correctly.""" reader = self.reader('local_includes') @@ -1060,7 +1059,7 @@ class TestEmitterBasic(unittest.TestCase): SandboxValidationError, 'Path specified in LOCAL_INCLUDES.*resolves to the ' 'topsrcdir or topobjdir'): - self.read_topsrcdir(reader) + objs = self.read_topsrcdir(reader) reader = self.reader('local_includes-invalid/objdir') @@ -1068,7 +1067,7 @@ class TestEmitterBasic(unittest.TestCase): SandboxValidationError, 'Path specified in LOCAL_INCLUDES.*resolves to the ' 'topsrcdir or topobjdir'): - self.read_topsrcdir(reader) + objs = self.read_topsrcdir(reader) def test_local_includes_file(self): """Test that a filename can't be used in LOCAL_INCLUDES.""" @@ -1077,7 +1076,7 @@ class TestEmitterBasic(unittest.TestCase): with self.assertRaisesRegexp( SandboxValidationError, 'Path specified in LOCAL_INCLUDES is a filename'): - self.read_topsrcdir(reader) + objs = self.read_topsrcdir(reader) def test_generated_includes(self): """Test that GENERATED_INCLUDES is emitted correctly.""" @@ -1137,21 +1136,21 @@ class TestEmitterBasic(unittest.TestCase): def test_xpidl_module_no_sources(self): """XPIDL_MODULE without XPIDL_SOURCES should be rejected.""" with self.assertRaisesRegexp(SandboxValidationError, 'XPIDL_MODULE ' - 'cannot be defined'): + 'cannot be defined'): reader = self.reader('xpidl-module-no-sources') self.read_topsrcdir(reader) def test_xpidl_module_no_sources(self): """Missing XPIDL_SOURCES should be rejected.""" with self.assertRaisesRegexp(SandboxValidationError, 'File .* ' - 'from XPIDL_SOURCES does not exist'): + 'from XPIDL_SOURCES does not exist'): reader = self.reader('missing-xpidl') self.read_topsrcdir(reader) def test_missing_local_includes(self): """LOCAL_INCLUDES containing non-existent directories should be rejected.""" with self.assertRaisesRegexp(SandboxValidationError, 'Path specified in ' - 'LOCAL_INCLUDES does not exist'): + 'LOCAL_INCLUDES does not exist'): reader = self.reader('missing-local-includes') self.read_topsrcdir(reader) @@ -1160,7 +1159,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('library-defines') objs = self.read_topsrcdir(reader) - libraries = [o for o in objs if isinstance(o, StaticLibrary)] + libraries = [o for o in objs if isinstance(o,StaticLibrary)] library_flags = [o for o in objs if isinstance(o, ComputedFlags) and 'LIBRARY_DEFINES' in o.flags] expected = { @@ -1335,6 +1334,7 @@ class TestEmitterBasic(unittest.TestCase): reader.config.substs['OBJ_SUFFIX'])), linkable.objs) + def test_unified_sources(self): """Test that UNIFIED_SOURCES works properly.""" reader = self.reader('unified-sources') @@ -1369,6 +1369,7 @@ class TestEmitterBasic(unittest.TestCase): reader.config.substs['OBJ_SUFFIX'])), linkable.objs) + def test_unified_sources_non_unified(self): """Test that UNIFIED_SOURCES with FILES_PER_UNIFIED_FILE=1 works properly.""" reader = self.reader('unified-sources-non-unified') @@ -1417,17 +1418,15 @@ class TestEmitterBasic(unittest.TestCase): def test_missing_final_target_pp_files(self): """Test that FINAL_TARGET_PP_FILES with missing files throws errors.""" with self.assertRaisesRegexp(SandboxValidationError, 'File listed in ' - 'FINAL_TARGET_PP_FILES does not exist'): + 'FINAL_TARGET_PP_FILES does not exist'): reader = self.reader('dist-files-missing') self.read_topsrcdir(reader) def test_final_target_pp_files_non_srcdir(self): '''Test that non-srcdir paths in FINAL_TARGET_PP_FILES throws errors.''' reader = self.reader('final-target-pp-files-non-srcdir') - with self.assertRaisesRegexp( - SandboxValidationError, - 'Only source directory paths allowed in FINAL_TARGET_PP_FILES:' - ): + with self.assertRaisesRegexp(SandboxValidationError, + 'Only source directory paths allowed in FINAL_TARGET_PP_FILES:'): self.read_topsrcdir(reader) def test_localized_files(self): @@ -1450,11 +1449,9 @@ class TestEmitterBasic(unittest.TestCase): """Test that LOCALIZED_FILES errors if a path does not start with `en-US/` or contain `locales/en-US/`.""" reader = self.reader('localized-files-no-en-us') - with self.assertRaisesRegexp( - SandboxValidationError, - 'LOCALIZED_FILES paths must start with `en-US/` or contain `locales/en-US/`: foo.js' - ): - self.read_topsrcdir(reader) + with self.assertRaisesRegexp(SandboxValidationError, + 'LOCALIZED_FILES paths must start with `en-US/` or contain `locales/en-US/`: foo.js'): + objs = self.read_topsrcdir(reader) def test_localized_pp_files(self): """Test that LOCALIZED_PP_FILES works properly.""" @@ -1476,28 +1473,28 @@ class TestEmitterBasic(unittest.TestCase): '''Test that defining a RustLibrary without a Cargo.toml fails.''' reader = self.reader('rust-library-no-cargo-toml') with self.assertRaisesRegexp(SandboxValidationError, - 'No Cargo.toml file found'): + 'No Cargo.toml file found'): self.read_topsrcdir(reader) def test_rust_library_name_mismatch(self): '''Test that defining a RustLibrary that doesn't match Cargo.toml fails.''' reader = self.reader('rust-library-name-mismatch') with self.assertRaisesRegexp(SandboxValidationError, - 'library.*does not match Cargo.toml-defined package'): + 'library.*does not match Cargo.toml-defined package'): self.read_topsrcdir(reader) def test_rust_library_no_lib_section(self): '''Test that a RustLibrary Cargo.toml with no [lib] section fails.''' reader = self.reader('rust-library-no-lib-section') with self.assertRaisesRegexp(SandboxValidationError, - 'Cargo.toml for.* has no \\[lib\\] section'): + 'Cargo.toml for.* has no \\[lib\\] section'): self.read_topsrcdir(reader) def test_rust_library_invalid_crate_type(self): '''Test that a RustLibrary Cargo.toml has a permitted crate-type.''' reader = self.reader('rust-library-invalid-crate-type') with self.assertRaisesRegexp(SandboxValidationError, - 'crate-type.* is not permitted'): + 'crate-type.* is not permitted'): self.read_topsrcdir(reader) def test_rust_library_dash_folding(self): @@ -1520,7 +1517,7 @@ class TestEmitterBasic(unittest.TestCase): reader = self.reader('multiple-rust-libraries', extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc')) with self.assertRaisesRegexp(LinkageMultipleRustLibrariesError, - 'Cannot link multiple Rust libraries'): + 'Cannot link multiple Rust libraries'): self.read_topsrcdir(reader) def test_rust_library_features(self): @@ -1540,21 +1537,21 @@ class TestEmitterBasic(unittest.TestCase): '''Test that duplicate RustLibrary features are rejected.''' reader = self.reader('rust-library-duplicate-features') with self.assertRaisesRegexp(SandboxValidationError, - 'features for .* should not contain duplicates'): + 'features for .* should not contain duplicates'): self.read_topsrcdir(reader) def test_rust_program_no_cargo_toml(self): '''Test that specifying RUST_PROGRAMS without a Cargo.toml fails.''' reader = self.reader('rust-program-no-cargo-toml') with self.assertRaisesRegexp(SandboxValidationError, - 'No Cargo.toml file found'): + 'No Cargo.toml file found'): self.read_topsrcdir(reader) def test_host_rust_program_no_cargo_toml(self): '''Test that specifying HOST_RUST_PROGRAMS without a Cargo.toml fails.''' reader = self.reader('host-rust-program-no-cargo-toml') with self.assertRaisesRegexp(SandboxValidationError, - 'No Cargo.toml file found'): + 'No Cargo.toml file found'): self.read_topsrcdir(reader) def test_rust_program_nonexistent_name(self): @@ -1562,7 +1559,7 @@ class TestEmitterBasic(unittest.TestCase): correctly throws an error.''' reader = self.reader('rust-program-nonexistent-name') with self.assertRaisesRegexp(SandboxValidationError, - 'Cannot find Cargo.toml definition for'): + 'Cannot find Cargo.toml definition for'): self.read_topsrcdir(reader) def test_host_rust_program_nonexistent_name(self): @@ -1570,7 +1567,7 @@ class TestEmitterBasic(unittest.TestCase): Cargo.toml correctly throws an error.''' reader = self.reader('host-rust-program-nonexistent-name') with self.assertRaisesRegexp(SandboxValidationError, - 'Cannot find Cargo.toml definition for'): + 'Cannot find Cargo.toml definition for'): self.read_topsrcdir(reader) def test_rust_programs(self): @@ -1673,10 +1670,8 @@ class TestEmitterBasic(unittest.TestCase): from GENERATED_FILES is an error. """ reader = self.reader('test-symbols-file-objdir-missing-generated') - with self.assertRaisesRegexp( - SandboxValidationError, - 'Objdir file specified in SYMBOLS_FILE not in GENERATED_FILES:' - ): + with self.assertRaisesRegexp(SandboxValidationError, + 'Objdir file specified in SYMBOLS_FILE not in GENERATED_FILES:'): self.read_topsrcdir(reader) diff --git a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py index 630c94b43ad5..71cc634e1bdb 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py +++ b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import unittest @@ -52,7 +52,7 @@ VARIABLES = { 'FUGA': (Fuga, unicode, None), 'PIYO': (Piyo, unicode, None), 'HOGERA': (ContextDerivedTypedList(Piyo, StrictOrderingOnAppendList), - list, None), + list, None), 'HOGEHOGE': (ContextDerivedTypedListWithItems( Piyo, StrictOrderingOnAppendListWithFlagsFactory({ @@ -60,7 +60,6 @@ VARIABLES = { })), list, None), } - class TestContext(unittest.TestCase): def test_key_rejection(self): # Lowercase keys should be rejected during normal operation. @@ -204,6 +203,5 @@ class TestContext(unittest.TestCase): with self.assertRaises(UnsortedError): ns['HOGEHOGE'] += ['f', 'e', 'd'] - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py index a8997e0727ec..dc4edb69d63c 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_reader.py +++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import sys @@ -72,7 +72,7 @@ class TestBuildReader(unittest.TestCase): self.assertTrue(os.path.exists(path)) contexts = list(reader.read_mozbuild(path, reader.config, - descend=False)) + descend=False)) self.assertEqual(len(contexts), 1) @@ -116,7 +116,7 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertEqual(e.actual_file, self.file_path('reader-error-basic', - 'moz.build')) + 'moz.build')) self.assertIn('The error occurred while processing the', str(e)) @@ -128,9 +128,9 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertEqual(e.actual_file, - self.file_path('reader-error-included-from', 'child.build')) + self.file_path('reader-error-included-from', 'child.build')) self.assertEqual(e.main_file, - self.file_path('reader-error-included-from', 'moz.build')) + self.file_path('reader-error-included-from', 'moz.build')) self.assertIn('This file was included as part of processing', str(e)) @@ -176,13 +176,13 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertIn('The error was triggered on line 5', str(e)) self.assertIn('is an attempt to write an illegal value to a special', - str(e)) + str(e)) self.assertIn('variable whose value was rejected is:\n\n DIRS', - str(e)) + str(e)) self.assertIn('written to it was of the following type:\n\n %s' % text_type, - str(e)) + str(e)) self.assertIn('expects the following type(s):\n\n list', str(e)) @@ -194,7 +194,7 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertIn('The underlying problem is an illegal file access', - str(e)) + str(e)) def test_error_missing_include_path(self): reader = self.reader('reader-error-missing-include') @@ -213,7 +213,7 @@ class TestBuildReader(unittest.TestCase): e = bre.exception self.assertIn('The error appears to be the fault of the script', - str(e)) + str(e)) self.assertIn(' ["TypeError: unsupported operand', str(e)) def test_error_bad_dir(self): @@ -247,7 +247,7 @@ class TestBuildReader(unittest.TestCase): def test_error_error_func_ok(self): reader = self.reader('reader-error-error-func', error_is_fatal=False) - list(reader.read_topsrcdir()) + contexts = list(reader.read_topsrcdir()) def test_error_empty_list(self): reader = self.reader('reader-error-empty-list') @@ -265,9 +265,9 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(len(contexts), 4) self.assertEqual([context.relsrcdir for context in contexts], - ['', 'foo', 'foo/baz', 'bar']) + ['', 'foo', 'foo/baz', 'bar']) self.assertEqual([context['XPIDL_MODULE'] for context in contexts], - ['foobar', 'foobar', 'baz', 'foobar']) + ['foobar', 'foobar', 'baz', 'foobar']) def test_find_relevant_mozbuilds(self): reader = self.reader('reader-relevant-mozbuild') @@ -334,16 +334,16 @@ class TestBuildReader(unittest.TestCase): reader = self.reader('reader-relevant-mozbuild') paths, contexts = reader.read_relevant_mozbuilds(['d1/every-level/a/file', - 'd1/every-level/b/file', 'd2/file']) + 'd1/every-level/b/file', 'd2/file']) self.assertEqual(len(paths), 3) self.assertEqual(len(contexts), 6) self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/a/file']], - ['', 'd1', 'd1/every-level', 'd1/every-level/a']) + ['', 'd1', 'd1/every-level', 'd1/every-level/a']) self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/b/file']], - ['', 'd1', 'd1/every-level', 'd1/every-level/b']) + ['', 'd1', 'd1/every-level', 'd1/every-level/b']) self.assertEqual([ctx.relsrcdir for ctx in paths['d2/file']], - ['', 'd2']) + ['', 'd2']) def test_files_bad_bug_component(self): reader = self.reader('files-info') @@ -389,8 +389,7 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(js_flags['BUG_COMPONENT'], BugzillaComponent('Firefox', 'JS')) self.assertEqual(cpp_flags['BUG_COMPONENT'], BugzillaComponent('Firefox', 'C++')) - self.assertEqual(misc_flags['BUG_COMPONENT'], BugzillaComponent( - 'default_product', 'default_component')) + self.assertEqual(misc_flags['BUG_COMPONENT'], BugzillaComponent('default_product', 'default_component')) def test_files_bug_component_final(self): reader = self.reader('files-info') @@ -402,13 +401,13 @@ class TestBuildReader(unittest.TestCase): 'bug_component/final/subcomponent/bar']) self.assertEqual(v['bug_component/final/foo']['BUG_COMPONENT'], - BugzillaComponent('default_product', 'default_component')) + BugzillaComponent('default_product', 'default_component')) self.assertEqual(v['bug_component/final/Makefile.in']['BUG_COMPONENT'], - BugzillaComponent('Firefox Build System', 'General')) + BugzillaComponent('Firefox Build System', 'General')) self.assertEqual(v['bug_component/final/subcomponent/Makefile.in']['BUG_COMPONENT'], - BugzillaComponent('Firefox Build System', 'General')) + BugzillaComponent('Firefox Build System', 'General')) self.assertEqual(v['bug_component/final/subcomponent/bar']['BUG_COMPONENT'], - BugzillaComponent('Another', 'Component')) + BugzillaComponent('Another', 'Component')) def test_file_test_deps(self): reader = self.reader('files-test-metadata') @@ -507,8 +506,7 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(info['subd/aa.py']['SCHEDULES'].exclusive, schedules.EXCLUSIVE_COMPONENTS) # Files('yaml.py') in subd/moz.build combines with Files('subdir/**.py') self.assertEqual(info['subd/yaml.py']['SCHEDULES'].inclusive, ['py-lint', 'yaml-lint']) - self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, - schedules.EXCLUSIVE_COMPONENTS) + self.assertEqual(info['subd/yaml.py']['SCHEDULES'].exclusive, schedules.EXCLUSIVE_COMPONENTS) # .. but exlusive does not override inclusive self.assertEqual(info['subd/win.js']['SCHEDULES'].inclusive, ['js-lint']) self.assertEqual(info['subd/win.js']['SCHEDULES'].exclusive, ['windows']) @@ -521,6 +519,5 @@ class TestBuildReader(unittest.TestCase): self.assertEqual(set(info['win.and.osx']['SCHEDULES'].exclusive), set(['macosx', 'windows'])) - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py index aa66a72c6cf0..d24c5d9ea624 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py +++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py @@ -2,8 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals +import os +import shutil import unittest from mozunit import main @@ -28,6 +30,7 @@ from mozbuild.frontend.context import ( ) from mozbuild.test.common import MockConfig +from types import StringTypes import mozpack.path as mozpath @@ -96,7 +99,7 @@ class TestSandbox(unittest.TestCase): sandbox.exec_source('DIRS = ["foo"]') with self.assertRaises(SandboxExecutionError) as se: - sandbox.exec_source('DIRS = ["bar"]') + sandbox.exec_source('DIRS = ["bar"]') self.assertEqual(sandbox['DIRS'], ['foo']) e = se.exception @@ -126,7 +129,6 @@ class TestedSandbox(MozbuildSandbox): It automatically normalizes paths given to exec_file and exec_source. This helps simplify the test code. ''' - def normalize_path(self, path): return mozpath.normpath( mozpath.join(self._context.config.topsrcdir, path)) @@ -139,7 +141,7 @@ class TestedSandbox(MozbuildSandbox): def exec_source(self, source, path=''): super(TestedSandbox, self).exec_source(source, - self.normalize_path(path) if path else '') + self.normalize_path(path) if path else '') class TestMozbuildSandbox(unittest.TestCase): @@ -186,9 +188,9 @@ class TestMozbuildSandbox(unittest.TestCase): self.assertEqual(sandbox['TOPOBJDIR'], config.topobjdir) self.assertEqual(sandbox['RELATIVEDIR'], 'foo/bar') self.assertEqual(sandbox['SRCDIR'], - mozpath.join(config.topsrcdir, 'foo/bar')) + mozpath.join(config.topsrcdir, 'foo/bar')) self.assertEqual(sandbox['OBJDIR'], - mozpath.join(config.topobjdir, 'foo/bar')) + mozpath.join(config.topobjdir, 'foo/bar')) def test_config_access(self): sandbox = self.sandbox() @@ -236,7 +238,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox.exec_source('DIST_SUBDIR = "foo"') with self.assertRaises(SandboxExecutionError) as se: - sandbox.exec_source('DIST_SUBDIR = "bar"') + sandbox.exec_source('DIST_SUBDIR = "bar"') self.assertEqual(sandbox['DIST_SUBDIR'], 'foo') e = se.exception @@ -257,7 +259,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox.source_path('bar'), ]) self.assertEqual(sandbox._context.main_path, - sandbox.normalize_path('moz.build')) + sandbox.normalize_path('moz.build')) self.assertEqual(len(sandbox._context.all_paths), 2) def test_include_outside_topsrcdir(self): @@ -267,7 +269,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox.exec_file('relative.build') self.assertEqual(se.exception.illegal_path, - sandbox.normalize_path('../moz.build')) + sandbox.normalize_path('../moz.build')) def test_include_error_stack(self): # Ensure the path stack is reported properly in exceptions. @@ -343,7 +345,7 @@ class TestMozbuildSandbox(unittest.TestCase): sandbox = MozbuildSandbox(Context(VARIABLES, config)) self.assertEqual(sandbox['CONFIG']['BAD_UTF8'], - u'\ufffd\ufffd\ufffd\ufffd:') + u'\ufffd\ufffd\ufffd\ufffd:') def test_invalid_exports_set_base(self): sandbox = self.sandbox() @@ -470,7 +472,7 @@ def foo(): e = se.exception.exc_value self.assertEqual(e.message, - 'Template function names must be CamelCase.') + 'Template function names must be CamelCase.') # Template names must not already be registered. sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) @@ -487,12 +489,11 @@ def Template(): e = se.exception.exc_value self.assertEqual(e.message, - 'A template named "Template" was already declared in %s.' % - sandbox.normalize_path('templates.mozbuild')) + 'A template named "Template" was already declared in %s.' % + sandbox.normalize_path('templates.mozbuild')) def test_function_args(self): - class Foo(int): - pass + class Foo(int): pass def foo(a, b): return type(a), type(b) diff --git a/python/mozbuild/mozbuild/test/test_android_version_code.py b/python/mozbuild/mozbuild/test/test_android_version_code.py index ff1cf6c554f2..80fa91ad69a3 100644 --- a/python/mozbuild/mozbuild/test/test_android_version_code.py +++ b/python/mozbuild/mozbuild/test/test_android_version_code.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozunit import main import unittest @@ -12,47 +10,38 @@ from mozbuild.android_version_code import ( android_version_code_v1, ) - class TestAndroidVersionCode(unittest.TestCase): def test_android_version_code_v0(self): # From https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&revision=e25de9972a77. buildid = '20150708104620' - arm_api9 = 2015070819 + arm_api9 = 2015070819 arm_api11 = 2015070821 - x86_api9 = 2015070822 - self.assertEqual(android_version_code_v0( - buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) - self.assertEqual(android_version_code_v0( - buildid, cpu_arch='armeabi-v7a', min_sdk=11, max_sdk=None), arm_api11) - self.assertEqual(android_version_code_v0( - buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) + x86_api9 = 2015070822 + self.assertEqual(android_version_code_v0(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) + self.assertEqual(android_version_code_v0(buildid, cpu_arch='armeabi-v7a', min_sdk=11, max_sdk=None), arm_api11) + self.assertEqual(android_version_code_v0(buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) def test_android_version_code_v1(self): buildid = '20150825141628' - arm_api16 = 0b01111000001000000001001001110001 + arm_api16 = 0b01111000001000000001001001110001 arm64_api21 = 0b01111000001000000001001001110100 - x86_api9 = 0b01111000001000000001001001110100 - self.assertEqual(android_version_code_v1( - buildid, cpu_arch='armeabi-v7a', min_sdk=16, max_sdk=None), arm_api16) - self.assertEqual(android_version_code_v1( - buildid, cpu_arch='arm64-v8a', min_sdk=21, max_sdk=None), arm64_api21) - self.assertEqual(android_version_code_v1( - buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) + x86_api9 = 0b01111000001000000001001001110100 + self.assertEqual(android_version_code_v1(buildid, cpu_arch='armeabi-v7a', min_sdk=16, max_sdk=None), arm_api16) + self.assertEqual(android_version_code_v1(buildid, cpu_arch='arm64-v8a', min_sdk=21, max_sdk=None), arm64_api21) + self.assertEqual(android_version_code_v1(buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) def test_android_version_code_v1_underflow(self): '''Verify that it is an error to ask for v1 codes predating the cutoff.''' - buildid = '201508010000' # Earliest possible. + buildid = '201508010000' # Earliest possible. arm_api9 = 0b01111000001000000000000000000000 - self.assertEqual(android_version_code_v1( - buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) + self.assertEqual(android_version_code_v1(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) with self.assertRaises(ValueError) as cm: - underflow = '201507310000' # Latest possible (valid) underflowing date. + underflow = '201507310000' # Latest possible (valid) underflowing date. android_version_code_v1(underflow, cpu_arch='armeabi', min_sdk=9, max_sdk=None) self.assertTrue('underflow' in cm.exception.message) def test_android_version_code_v1_running_low(self): - '''Verify there is an informative message if one asks for v1 - codes that are close to overflow.''' + '''Verify there is an informative message if one asks for v1 codes that are close to overflow.''' with self.assertRaises(ValueError) as cm: overflow = '20290801000000' android_version_code_v1(overflow, cpu_arch='armeabi', min_sdk=9, max_sdk=None) @@ -68,9 +57,8 @@ class TestAndroidVersionCode(unittest.TestCase): def test_android_version_code_v0_relative_v1(self): '''Verify that the first v1 code is greater than the equivalent v0 code.''' buildid = '20150801000000' - self.assertGreater( - android_version_code_v1(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), - android_version_code_v0(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None)) + self.assertGreater(android_version_code_v1(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), + android_version_code_v0(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None)) if __name__ == '__main__': diff --git a/python/mozbuild/mozbuild/test/test_artifact_cache.py b/python/mozbuild/mozbuild/test/test_artifact_cache.py index 4648f5c3e7af..780473a0462e 100644 --- a/python/mozbuild/mozbuild/test/test_artifact_cache.py +++ b/python/mozbuild/mozbuild/test/test_artifact_cache.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import mozunit @@ -24,7 +24,6 @@ CONTENTS = { 'http://server/larger': b'larger' * 3000, } - class FakeResponse(object): def __init__(self, content): self._content = content diff --git a/python/mozbuild/mozbuild/test/test_base.py b/python/mozbuild/mozbuild/test/test_base.py index bb08572bb2b7..e728bfa395ec 100644 --- a/python/mozbuild/mozbuild/test/test_base.py +++ b/python/mozbuild/mozbuild/test/test_base.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import json import os @@ -23,6 +23,7 @@ from mozbuild.base import ( BadEnvironmentException, MachCommandBase, MozbuildObject, + ObjdirMismatchException, PathArgument, ) @@ -75,7 +76,7 @@ class TestMozbuildObject(unittest.TestCase): os.environ[b'MOZCONFIG'] = mozconfig.name self.assertEqual(base.topobjdir, mozpath.join(base.topsrcdir, - 'foo')) + 'foo')) self.assertTrue(base.topobjdir.endswith('foo')) def test_objdir_config_status(self): @@ -213,7 +214,7 @@ class TestMozbuildObject(unittest.TestCase): context.topdir = topsrcdir context.settings = None context.log_manager = None - context.detect_virtualenv_mozinfo = False + context.detect_virtualenv_mozinfo=False o = MachCommandBase(context) @@ -279,7 +280,7 @@ class TestMozbuildObject(unittest.TestCase): context.topdir = topsrcdir context.settings = None context.log_manager = None - context.detect_virtualenv_mozinfo = False + context.detect_virtualenv_mozinfo=False stdout = sys.stdout sys.stdout = StringIO() @@ -331,7 +332,7 @@ class TestMozbuildObject(unittest.TestCase): substs.append(('BIN_SUFFIX', '')) base._config_environment = ConfigEnvironment(base.topsrcdir, - base.topobjdir, substs=substs) + base.topobjdir, substs=substs) p = base.get_binary_path('xpcshell', False) if platform.startswith('darwin'): @@ -365,7 +366,6 @@ class TestMozbuildObject(unittest.TestCase): else: self.assertTrue(p.endswith('foobar')) - class TestPathArgument(unittest.TestCase): def test_path_argument(self): # Absolute path @@ -404,6 +404,5 @@ class TestPathArgument(unittest.TestCase): self.assertEqual(p.srcdir_path(), "/src/foo/bar") self.assertEqual(p.objdir_path(), "/src/obj/foo/bar") - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/mozbuild/test/test_containers.py index ce92c13af092..3d46f86a90fc 100644 --- a/python/mozbuild/mozbuild/test/test_containers.py +++ b/python/mozbuild/mozbuild/test/test_containers.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest from mozunit import main @@ -32,7 +30,7 @@ class TestReadOnlyNamespace(unittest.TestCase): ['bar', 'foo']) with self.assertRaises(AttributeError): - test.missing + value = test.missing with self.assertRaises(Exception): test.foo = 2 @@ -58,7 +56,7 @@ class TestReadOnlyDict(unittest.TestCase): self.assertEqual(test['foo'], 1) with self.assertRaises(KeyError): - test['missing'] + value = test['missing'] with self.assertRaises(Exception): test['baz'] = True @@ -144,7 +142,6 @@ class TestList(unittest.TestCase): with self.assertRaises(ValueError): test = test + False - class TestOrderedDefaultDict(unittest.TestCase): def test_simple(self): original = OrderedDict(foo=1, bar=2) @@ -155,21 +152,21 @@ class TestOrderedDefaultDict(unittest.TestCase): self.assertEqual(test['foo'], 1) - self.assertEqual(test.keys(), ['foo', 'bar']) + self.assertEqual(test.keys(), ['foo', 'bar' ]) def test_defaults(self): - test = OrderedDefaultDict(bool, {'foo': 1}) + test = OrderedDefaultDict(bool, {'foo': 1 }) self.assertEqual(test['foo'], 1) self.assertEqual(test['qux'], False) - self.assertEqual(test.keys(), ['foo', 'qux']) + self.assertEqual(test.keys(), ['foo', 'qux' ]) class TestKeyedDefaultDict(unittest.TestCase): def test_simple(self): - original = {'foo': 1, 'bar': 2} + original = {'foo': 1, 'bar': 2 } test = KeyedDefaultDict(lambda x: x, original) @@ -178,7 +175,7 @@ class TestKeyedDefaultDict(unittest.TestCase): self.assertEqual(test['foo'], 1) def test_defaults(self): - test = KeyedDefaultDict(lambda x: x, {'foo': 1}) + test = KeyedDefaultDict(lambda x: x, {'foo': 1 }) self.assertEqual(test['foo'], 1) @@ -199,7 +196,7 @@ class TestKeyedDefaultDict(unittest.TestCase): class TestReadOnlyKeyedDefaultDict(unittest.TestCase): def test_defaults(self): - test = ReadOnlyKeyedDefaultDict(lambda x: x, {'foo': 1}) + test = ReadOnlyKeyedDefaultDict(lambda x: x, {'foo': 1 }) self.assertEqual(test['foo'], 1) diff --git a/python/mozbuild/mozbuild/test/test_dotproperties.py b/python/mozbuild/mozbuild/test/test_dotproperties.py index 2fde20538795..a03f85b0d179 100644 --- a/python/mozbuild/mozbuild/test/test_dotproperties.py +++ b/python/mozbuild/mozbuild/test/test_dotproperties.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import unittest @@ -31,6 +31,7 @@ key=value self.assertEqual(p.get('missing', 'default'), 'default') self.assertEqual(p.get('key'), 'value') + def test_update(self): contents = StringIO(''' old=old value @@ -47,6 +48,7 @@ key=new value self.assertEqual(p.get('old'), 'old value') self.assertEqual(p.get('key'), 'new value') + def test_get_list(self): contents = StringIO(''' list.0=A @@ -62,6 +64,7 @@ order.2=C self.assertEqual(p.get_list('list'), ['A', 'B', 'C']) self.assertEqual(p.get_list('order'), ['A', 'B', 'C']) + def test_get_list_with_shared_prefix(self): contents = StringIO(''' list.0=A @@ -82,6 +85,7 @@ list.other.0=H self.assertEqual(p.get_list('list.sublist.second'), ['G']) self.assertEqual(p.get_list('list.other'), ['H']) + def test_get_dict(self): contents = StringIO(''' A.title=title A @@ -104,6 +108,7 @@ C=value with self.assertRaises(ValueError): p.get_dict('C', required_keys=['missing_key']) + def test_get_dict_with_shared_prefix(self): contents = StringIO(''' A.title=title A @@ -118,9 +123,9 @@ B.subdict.url=url B subdict self.assertEqual(p.get_dict('A'), {'title': 'title A'}) self.assertEqual(p.get_dict('B'), {'title': 'title B', 'url': 'url B'}) self.assertEqual(p.get_dict('A.subdict'), - {'title': 'title A subdict'}) + {'title': 'title A subdict'}) self.assertEqual(p.get_dict('B.subdict'), - {'title': 'title B subdict', 'url': 'url B subdict'}) + {'title': 'title B subdict', 'url': 'url B subdict'}) def test_get_dict_with_value_prefix(self): contents = StringIO(''' @@ -138,6 +143,7 @@ A.default.C.ignored=C ignored self.assertEqual(p.get('A.default.B'), 'B') self.assertEqual(p.get('A.default.C'), 'C') + def test_unicode(self): contents = StringIO(''' # Danish. diff --git a/python/mozbuild/mozbuild/test/test_expression.py b/python/mozbuild/mozbuild/test/test_expression.py index ae6be66707c5..fb3c45894c81 100644 --- a/python/mozbuild/mozbuild/test/test_expression.py +++ b/python/mozbuild/mozbuild/test/test_expression.py @@ -1,85 +1,82 @@ -from __future__ import absolute_import, print_function - import unittest +import sys +import os.path import mozunit from mozbuild.preprocessor import Expression, Context - class TestContext(unittest.TestCase): - """ - Unit tests for the Context class - """ + """ + Unit tests for the Context class + """ - def setUp(self): - self.c = Context() - self.c['FAIL'] = 'PASS' + def setUp(self): + self.c = Context() + self.c['FAIL'] = 'PASS' - def test_string_literal(self): - """test string literal, fall-through for undefined var in a Context""" - self.assertEqual(self.c['PASS'], 'PASS') + def test_string_literal(self): + """test string literal, fall-through for undefined var in a Context""" + self.assertEqual(self.c['PASS'], 'PASS') - def test_variable(self): - """test value for defined var in the Context class""" - self.assertEqual(self.c['FAIL'], 'PASS') - - def test_in(self): - """test 'var in context' to not fall for fallback""" - self.assert_('FAIL' in self.c) - self.assert_('PASS' not in self.c) + def test_variable(self): + """test value for defined var in the Context class""" + self.assertEqual(self.c['FAIL'], 'PASS') + def test_in(self): + """test 'var in context' to not fall for fallback""" + self.assert_('FAIL' in self.c) + self.assert_('PASS' not in self.c) class TestExpression(unittest.TestCase): - """ - Unit tests for the Expression class - evaluate() is called with a context {FAIL: 'PASS'} - """ + """ + Unit tests for the Expression class + evaluate() is called with a context {FAIL: 'PASS'} + """ - def setUp(self): - self.c = Context() - self.c['FAIL'] = 'PASS' + def setUp(self): + self.c = Context() + self.c['FAIL'] = 'PASS' - def test_string_literal(self): - """Test for a string literal in an Expression""" - self.assertEqual(Expression('PASS').evaluate(self.c), 'PASS') + def test_string_literal(self): + """Test for a string literal in an Expression""" + self.assertEqual(Expression('PASS').evaluate(self.c), 'PASS') - def test_variable(self): - """Test for variable value in an Expression""" - self.assertEqual(Expression('FAIL').evaluate(self.c), 'PASS') + def test_variable(self): + """Test for variable value in an Expression""" + self.assertEqual(Expression('FAIL').evaluate(self.c), 'PASS') - def test_not(self): - """Test for the ! operator""" - self.assert_(Expression('!0').evaluate(self.c)) - self.assert_(not Expression('!1').evaluate(self.c)) + def test_not(self): + """Test for the ! operator""" + self.assert_(Expression('!0').evaluate(self.c)) + self.assert_(not Expression('!1').evaluate(self.c)) - def test_equals(self): - """ Test for the == operator""" - self.assert_(Expression('FAIL == PASS').evaluate(self.c)) + def test_equals(self): + """ Test for the == operator""" + self.assert_(Expression('FAIL == PASS').evaluate(self.c)) - def test_notequals(self): - """ Test for the != operator""" - self.assert_(Expression('FAIL != 1').evaluate(self.c)) + def test_notequals(self): + """ Test for the != operator""" + self.assert_(Expression('FAIL != 1').evaluate(self.c)) - def test_logical_and(self): - """ Test for the && operator""" - self.assertTrue(Expression('PASS == PASS && PASS != NOTPASS').evaluate(self.c)) + def test_logical_and(self): + """ Test for the && operator""" + self.assertTrue(Expression('PASS == PASS && PASS != NOTPASS').evaluate(self.c)) - def test_logical_or(self): - """ Test for the || operator""" - self.assertTrue(Expression('PASS == NOTPASS || PASS != NOTPASS').evaluate(self.c)) + def test_logical_or(self): + """ Test for the || operator""" + self.assertTrue(Expression('PASS == NOTPASS || PASS != NOTPASS').evaluate(self.c)) - def test_logical_ops(self): - """ Test for the && and || operators precedence""" - # Would evaluate to false if precedence was wrong - self.assertTrue(Expression( - 'PASS == PASS || PASS != NOTPASS && PASS == NOTPASS').evaluate(self.c)) + def test_logical_ops(self): + """ Test for the && and || operators precedence""" + # Would evaluate to false if precedence was wrong + self.assertTrue(Expression('PASS == PASS || PASS != NOTPASS && PASS == NOTPASS').evaluate(self.c)) - def test_defined(self): - """ Test for the defined() value""" - self.assertTrue(Expression('defined(FAIL)').evaluate(self.c)) - self.assertTrue(Expression('!defined(PASS)').evaluate(self.c)) + def test_defined(self): + """ Test for the defined() value""" + self.assertTrue(Expression('defined(FAIL)').evaluate(self.c)) + self.assertTrue(Expression('!defined(PASS)').evaluate(self.c)) if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_jarmaker.py b/python/mozbuild/mozbuild/test/test_jarmaker.py index 689b580f7825..1a01294cfb52 100644 --- a/python/mozbuild/mozbuild/test/test_jarmaker.py +++ b/python/mozbuild/mozbuild/test/test_jarmaker.py @@ -2,12 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function import unittest -import os -import sys -import os.path +import os, sys, os.path, time, inspect from filecmp import dircmp from tempfile import mkdtemp from shutil import rmtree, copy2 @@ -68,7 +66,6 @@ if sys.platform == "win32": DWORD] GetVolumeInformation.restype = ctypes.c_int - def symlinks_supported(path): if sys.platform == "win32": # Add 1 for a trailing backslash if necessary, and 1 for the terminating @@ -89,7 +86,6 @@ def symlinks_supported(path): else: return True - def _getfileinfo(path): """Return information for the given file. This only works on Windows.""" fh = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, OPEN_EXISTING, 0, None) @@ -101,7 +97,6 @@ def _getfileinfo(path): raise WinError() return info - def is_symlink_to(dest, src): if sys.platform == "win32": # Check if both are on the same volume and have the same file ID @@ -118,11 +113,9 @@ def is_symlink_to(dest, src): abssrc = os.path.abspath(src) return target == abssrc - class _TreeDiff(dircmp): """Helper to report rich results on difference between two directories. """ - def _fillDiff(self, dc, rv, basepath="{0}"): rv['right_only'] += map(lambda l: basepath.format(l), dc.right_only) rv['left_only'] += map(lambda l: basepath.format(l), dc.left_only) @@ -131,10 +124,9 @@ class _TreeDiff(dircmp): rv['funny'] += map(lambda l: basepath.format(l), dc.funny_files) for subdir, _dc in dc.subdirs.iteritems(): self._fillDiff(_dc, rv, basepath.format(subdir + "/{0}")) - def allResults(self, left, right): - rv = {'right_only': [], 'left_only': [], - 'diff_files': [], 'funny': []} + rv = {'right_only':[], 'left_only':[], + 'diff_files':[], 'funny': []} self._fillDiff(self, rv) chunks = [] if rv['right_only']: @@ -149,13 +141,11 @@ class _TreeDiff(dircmp): chunks.append("{0} don't compare".format(', '.join(rv['funny']))) return '; '.join(chunks) - class TestJarMaker(unittest.TestCase): """ Unit tests for JarMaker.py """ - debug = False # set to True to debug failing tests on disk - + debug = False # set to True to debug failing tests on disk def setUp(self): self.tmpdir = mkdtemp() self.srcdir = os.path.join(self.tmpdir, 'src') @@ -225,18 +215,18 @@ class TestJarMaker(unittest.TestCase): dir/foo (bar) ''') jarf.close() - open(os.path.join(self.srcdir, 'bar'), 'w').write('content\n') + open(os.path.join(self.srcdir,'bar'),'w').write('content\n') # create reference refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir') os.makedirs(refpath) - open(os.path.join(refpath, 'foo'), 'w').write('content\n') + open(os.path.join(refpath, 'foo'), 'w').write('content\n') def test_a_simple_jar(self): '''Test a simple jar.mn''' self._create_simple_setup() # call JarMaker - rv = self._jar_and_compare(os.path.join(self.srcdir, 'jar.mn'), - sourcedirs=[self.srcdir]) + rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'), + sourcedirs = [self.srcdir]) self.assertTrue(not rv, rv) def test_a_simple_symlink(self): @@ -248,7 +238,7 @@ class TestJarMaker(unittest.TestCase): jm = JarMaker(outputFormat='symlink') jm.sourcedirs = [self.srcdir] jm.topsourcedir = self.srcdir - jm.makeJar(os.path.join(self.srcdir, 'jar.mn'), self.builddir) + jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir) # All we do is check that srcdir/bar points to builddir/chrome/test/dir/foo srcbar = os.path.join(self.srcdir, 'bar') destfoo = os.path.join(self.builddir, 'chrome', 'test', 'dir', 'foo') @@ -263,12 +253,12 @@ class TestJarMaker(unittest.TestCase): dir/hoge (qux/*) ''') jarf.close() - open(os.path.join(self.srcdir, 'foo.js'), 'w').write('foo.js\n') - open(os.path.join(self.srcdir, 'bar.js'), 'w').write('bar.js\n') + open(os.path.join(self.srcdir,'foo.js'),'w').write('foo.js\n') + open(os.path.join(self.srcdir,'bar.js'),'w').write('bar.js\n') os.makedirs(os.path.join(self.srcdir, 'qux', 'foo')) - open(os.path.join(self.srcdir, 'qux', 'foo', '1'), 'w').write('1\n') - open(os.path.join(self.srcdir, 'qux', 'foo', '2'), 'w').write('2\n') - open(os.path.join(self.srcdir, 'qux', 'baz'), 'w').write('baz\n') + open(os.path.join(self.srcdir,'qux', 'foo', '1'),'w').write('1\n') + open(os.path.join(self.srcdir,'qux', 'foo', '2'),'w').write('2\n') + open(os.path.join(self.srcdir,'qux', 'baz'),'w').write('baz\n') # create reference refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir') os.makedirs(os.path.join(refpath, 'bar')) @@ -283,8 +273,8 @@ class TestJarMaker(unittest.TestCase): '''Test a wildcard in jar.mn''' self._create_wildcard_setup() # call JarMaker - rv = self._jar_and_compare(os.path.join(self.srcdir, 'jar.mn'), - sourcedirs=[self.srcdir]) + rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'), + sourcedirs = [self.srcdir]) self.assertTrue(not rv, rv) def test_a_wildcard_symlink(self): @@ -296,7 +286,7 @@ class TestJarMaker(unittest.TestCase): jm = JarMaker(outputFormat='symlink') jm.sourcedirs = [self.srcdir] jm.topsourcedir = self.srcdir - jm.makeJar(os.path.join(self.srcdir, 'jar.mn'), self.builddir) + jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir) expected_symlinks = { ('bar', 'foo.js'): ('foo.js',), @@ -321,11 +311,9 @@ class Test_relativesrcdir(unittest.TestCase): self.jm.relativesrcdir = 'browser/locales' self.fake_empty_file = StringIO() self.fake_empty_file.name = 'fake_empty_file' - def tearDown(self): del self.jm del self.fake_empty_file - def test_en_US(self): jm = self.jm jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') @@ -334,13 +322,11 @@ class Test_relativesrcdir(unittest.TestCase): os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'browser/locales', 'en-US') ]) - def test_l10n_no_merge(self): jm = self.jm jm.l10nbase = '/L10N_BASE' jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') self.assertEquals(jm.localedirs, [os.path.join('/L10N_BASE', 'browser')]) - def test_l10n_merge(self): jm = self.jm jm.l10nbase = '/L10N_BASE' @@ -352,7 +338,6 @@ class Test_relativesrcdir(unittest.TestCase): os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'browser/locales', 'en-US') ]) - def test_override(self): jm = self.jm jm.outputFormat = 'flat' # doesn't touch chrome dir without files @@ -366,7 +351,6 @@ relativesrcdir dom/locales: os.path.join(os.path.abspath('/TOPSOURCEDIR'), 'dom/locales', 'en-US') ]) - def test_override_l10n(self): jm = self.jm jm.l10nbase = '/L10N_BASE' diff --git a/python/mozbuild/mozbuild/test/test_licenses.py b/python/mozbuild/mozbuild/test/test_licenses.py index e139ef9f65b1..e3b2ae8abc9d 100644 --- a/python/mozbuild/mozbuild/test/test_licenses.py +++ b/python/mozbuild/mozbuild/test/test_licenses.py @@ -1,35 +1,37 @@ -from __future__ import absolute_import, print_function - import unittest +from StringIO import StringIO +import os +import sys +import os.path import mozunit +from mozbuild.preprocessor import Preprocessor + from mozbuild.vendor_rust import VendorRust - class TestLicenses(unittest.TestCase): - """ - Unit tests for the Rust Vendoring stuff - """ + """ + Unit tests for the Rust Vendoring stuff + """ - def setUp(self): - pass + def setUp(self): + pass - def tearDown(self): - pass - - def testLicense(self): - self.assertEquals(VendorRust.runtime_license('', 'Apache-2.0'), True) - self.assertEquals(VendorRust.runtime_license('', 'MIT'), True) - self.assertEquals(VendorRust.runtime_license('', 'GPL'), False) - self.assertEquals(VendorRust.runtime_license('', 'MIT /GPL'), True) - self.assertEquals(VendorRust.runtime_license('', 'GPL/ Proprietary'), False) - self.assertEquals(VendorRust.runtime_license('', 'GPL AND MIT'), False) - self.assertEquals(VendorRust.runtime_license('', 'ISC\tAND\tMIT'), False) - self.assertEquals(VendorRust.runtime_license('', 'GPL OR MIT'), True) - self.assertEquals(VendorRust.runtime_license('', 'ALLIGATOR MIT'), False) - pass + def tearDown(self): + pass + def testLicense(self): + self.assertEquals(VendorRust.runtime_license('', 'Apache-2.0'), True) + self.assertEquals(VendorRust.runtime_license('', 'MIT'), True) + self.assertEquals(VendorRust.runtime_license('', 'GPL'), False) + self.assertEquals(VendorRust.runtime_license('', 'MIT /GPL'), True) + self.assertEquals(VendorRust.runtime_license('', 'GPL/ Proprietary'), False) + self.assertEquals(VendorRust.runtime_license('', 'GPL AND MIT'), False) + self.assertEquals(VendorRust.runtime_license('', 'ISC\tAND\tMIT'), False) + self.assertEquals(VendorRust.runtime_license('', 'GPL OR MIT'), True) + self.assertEquals(VendorRust.runtime_license('', 'ALLIGATOR MIT'), False) + pass if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuild/mozbuild/test/test_line_endings.py index bb15b086e60e..565abc8c93ff 100644 --- a/python/mozbuild/mozbuild/test/test_line_endings.py +++ b/python/mozbuild/mozbuild/test/test_line_endings.py @@ -1,49 +1,46 @@ -from __future__ import absolute_import, print_function - import unittest from StringIO import StringIO import os +import sys import os.path import mozunit from mozbuild.preprocessor import Preprocessor - class TestLineEndings(unittest.TestCase): - """ - Unit tests for the Context class - """ + """ + Unit tests for the Context class + """ - def setUp(self): - self.pp = Preprocessor() - self.pp.out = StringIO() - self.tempnam = os.tempnam('.') + def setUp(self): + self.pp = Preprocessor() + self.pp.out = StringIO() + self.tempnam = os.tempnam('.') - def tearDown(self): - os.remove(self.tempnam) + def tearDown(self): + os.remove(self.tempnam) - def createFile(self, lineendings): - f = open(self.tempnam, 'wb') - for line, ending in zip(['a', '#literal b', 'c'], lineendings): - f.write(line+ending) - f.close() + def createFile(self, lineendings): + f = open(self.tempnam, 'wb') + for line, ending in zip(['a', '#literal b', 'c'], lineendings): + f.write(line+ending) + f.close() - def testMac(self): - self.createFile(['\x0D']*3) - self.pp.do_include(self.tempnam) - self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + def testMac(self): + self.createFile(['\x0D']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') - def testUnix(self): - self.createFile(['\x0A']*3) - self.pp.do_include(self.tempnam) - self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') - - def testWindows(self): - self.createFile(['\x0D\x0A']*3) - self.pp.do_include(self.tempnam) - self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + def testUnix(self): + self.createFile(['\x0A']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + def testWindows(self): + self.createFile(['\x0D\x0A']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') if __name__ == '__main__': - mozunit.main() + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_makeutil.py b/python/mozbuild/mozbuild/test/test_makeutil.py index e46cd22a2350..6fffa0e0e1fa 100644 --- a/python/mozbuild/mozbuild/test/test_makeutil.py +++ b/python/mozbuild/mozbuild/test/test_makeutil.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozbuild.makeutil import ( Makefile, read_dep_makefile, @@ -41,10 +39,10 @@ class TestMakefile(unittest.TestCase): rule.add_commands(['$(BAZ) -o $@ $<', '$(TOUCH) $@']) rule.dump(out) self.assertEqual(out.getvalue(), - 'foo bar: baz\n' + - '\techo $@\n' + - '\t$(BAZ) -o $@ $<\n' + - '\t$(TOUCH) $@\n') + 'foo bar: baz\n' + + '\techo $@\n' + + '\t$(BAZ) -o $@ $<\n' + + '\t$(TOUCH) $@\n') out.truncate(0) rule = Rule(['foo']) @@ -98,19 +96,19 @@ class TestMakefile(unittest.TestCase): rule.add_commands(['echo $@']) mk.dump(out, removal_guard=False) self.assertEqual(out.getvalue(), - 'foo: bar baz qux\n' + - '\techo foo\n' + - 'bar baz: hoge\n' + - '\techo $@\n') + 'foo: bar baz qux\n' + + '\techo foo\n' + + 'bar baz: hoge\n' + + '\techo $@\n') out.truncate(0) mk.dump(out) self.assertEqual(out.getvalue(), - 'foo: bar baz qux\n' + - '\techo foo\n' + - 'bar baz: hoge\n' + - '\techo $@\n' + - 'hoge qux:\n') + 'foo: bar baz qux\n' + + '\techo foo\n' + + 'bar baz: hoge\n' + + '\techo $@\n' + + 'hoge qux:\n') def test_statement(self): out = StringIO() @@ -121,11 +119,11 @@ class TestMakefile(unittest.TestCase): mk.create_rule(['$(BAR)']).add_commands(['echo $@']) mk.dump(out, removal_guard=False) self.assertEqual(out.getvalue(), - 'foo: bar\n' + - '\techo foo\n' + - 'BAR = bar\n' + - '$(BAR):\n' + - '\techo $@\n') + 'foo: bar\n' + + '\techo foo\n' + + 'BAR = bar\n' + + '$(BAR):\n' + + '\techo $@\n') @unittest.skipIf(os.name != 'nt', 'Test only applicable on Windows.') def test_path_normalization(self): @@ -136,9 +134,9 @@ class TestMakefile(unittest.TestCase): rule.add_commands(['echo c:\\foo']) mk.dump(out) self.assertEqual(out.getvalue(), - 'c:/foo: c:/bar c:/baz/qux\n' + - '\techo c:\\foo\n' + - 'c:/bar c:/baz/qux:\n') + 'c:/foo: c:/bar c:/baz/qux\n' + + '\techo c:\\foo\n' + + 'c:/bar c:/baz/qux:\n') def test_read_dep_makefile(self): input = StringIO( @@ -163,6 +161,5 @@ class TestMakefile(unittest.TestCase): 'target: b c a\n' + 'a b c:\n') - if __name__ == '__main__': main() diff --git a/python/mozbuild/mozbuild/test/test_manifest.py b/python/mozbuild/mozbuild/test/test_manifest.py index d61eeabc1f3e..38ab4763072f 100644 --- a/python/mozbuild/mozbuild/test/test_manifest.py +++ b/python/mozbuild/mozbuild/test/test_manifest.py @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import unittest diff --git a/python/mozbuild/mozbuild/test/test_mozconfig.py b/python/mozbuild/mozbuild/test/test_mozconfig.py index 0f7e4428d69f..7178b8942796 100644 --- a/python/mozbuild/mozbuild/test/test_mozconfig.py +++ b/python/mozbuild/mozbuild/test/test_mozconfig.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os import unittest @@ -139,6 +139,7 @@ class TestMozconfigLoader(unittest.TestCase): srcdir = self.get_temp_dir() curdir = self.get_temp_dir() + dirs = [srcdir, curdir] loader = MozconfigLoader(srcdir) path = os.path.join(srcdir, relative_mozconfig) @@ -198,7 +199,7 @@ class TestMozconfigLoader(unittest.TestCase): MozconfigLoader(d).find_mozconfig() self.assertIn('Multiple default mozconfig files present', - e.exception.message) + e.exception.message) def test_find_deprecated_path_srcdir(self): """Ensure we error when deprecated path locations are present.""" @@ -211,7 +212,7 @@ class TestMozconfigLoader(unittest.TestCase): MozconfigLoader(d).find_mozconfig() self.assertIn('This implicit location is no longer', - e.exception.message) + e.exception.message) self.assertIn(d, e.exception.message) def test_find_deprecated_home_paths(self): @@ -229,7 +230,7 @@ class TestMozconfigLoader(unittest.TestCase): self.get_loader().find_mozconfig() self.assertIn('This implicit location is no longer', - e.exception.message) + e.exception.message) self.assertIn(path, e.exception.message) def test_read_no_mozconfig(self): @@ -326,7 +327,7 @@ class TestMozconfigLoader(unittest.TestCase): result = loader.read_mozconfig(mozconfig.name) self.assertEqual(result['topobjdir'], '%s/some-objdir' % - loader.topsrcdir) + loader.topsrcdir) def test_read_new_variables(self): """New variables declared in mozconfig file are detected.""" @@ -426,7 +427,7 @@ class TestMozconfigLoader(unittest.TestCase): result = loader.read_mozconfig(mozconfig.name) self.assertEqual(result['vars']['added']['TEST'], - loader.topsrcdir.replace(os.sep, '/')) + loader.topsrcdir.replace(os.sep, '/')) self.assertEqual(result['env']['added'], {}) def test_read_empty_variable_value(self): @@ -458,7 +459,7 @@ class TestMozconfigLoader(unittest.TestCase): self.assertTrue(e.exception.message.startswith( 'Evaluation of your mozconfig exited with an error')) self.assertEquals(e.exception.path, - mozconfig.name.replace(os.sep, '/')) + mozconfig.name.replace(os.sep, '/')) self.assertEquals(e.exception.output, ['hello world']) diff --git a/python/mozbuild/mozbuild/test/test_mozinfo.py b/python/mozbuild/mozbuild/test/test_mozinfo.py index 4ebe3a99a5c8..6ae3da566ec4 100755 --- a/python/mozbuild/mozbuild/test/test_mozinfo.py +++ b/python/mozbuild/mozbuild/test/test_mozinfo.py @@ -3,8 +3,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import json import os import tempfile @@ -203,7 +201,6 @@ class TestWriteMozinfo(unittest.TestCase, Base): """ Test the write_mozinfo function. """ - def setUp(self): fd, self.f = tempfile.mkstemp() os.close(fd) diff --git a/python/mozbuild/mozbuild/test/test_preprocessor.py b/python/mozbuild/mozbuild/test/test_preprocessor.py index cb9895f9e24a..e17a24f66a2e 100644 --- a/python/mozbuild/mozbuild/test/test_preprocessor.py +++ b/python/mozbuild/mozbuild/test/test_preprocessor.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest from StringIO import StringIO diff --git a/python/mozbuild/mozbuild/test/test_pythonutil.py b/python/mozbuild/mozbuild/test/test_pythonutil.py index 2aed259958b9..a11fce84561d 100644 --- a/python/mozbuild/mozbuild/test/test_pythonutil.py +++ b/python/mozbuild/mozbuild/test/test_pythonutil.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozbuild.pythonutil import iter_modules_in_path from mozunit import main import os diff --git a/python/mozbuild/mozbuild/test/test_util.py b/python/mozbuild/mozbuild/test/test_util.py index 71309365b08f..cc0761e0e21a 100644 --- a/python/mozbuild/mozbuild/test/test_util.py +++ b/python/mozbuild/mozbuild/test/test_util.py @@ -3,7 +3,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import itertools import hashlib @@ -108,7 +108,6 @@ class TestFileAvoidWrite(unittest.TestCase): '''MockedOpen extension to raise an exception if something attempts to write in an opened file. ''' - def __call__(self, name, mode): if 'w' in mode: raise Exception('Unexpected open with write mode') @@ -162,7 +161,6 @@ class TestFileAvoidWrite(unittest.TestCase): finally: shutil.rmtree(tmpdir) - class TestResolveTargetToMake(unittest.TestCase): def setUp(self): self.topobjdir = data_path @@ -197,25 +195,20 @@ class TestResolveTargetToMake(unittest.TestCase): def test_regular_file(self): self.assertResolve('test-dir/with/file', ('test-dir/with', 'file')) self.assertResolve('test-dir/with/without/file', ('test-dir/with', 'without/file')) - self.assertResolve('test-dir/with/without/with/file', - ('test-dir/with/without/with', 'file')) + self.assertResolve('test-dir/with/without/with/file', ('test-dir/with/without/with', 'file')) self.assertResolve('test-dir/without/file', ('test-dir', 'without/file')) self.assertResolve('test-dir/without/with/file', ('test-dir/without/with', 'file')) - self.assertResolve('test-dir/without/with/without/file', - ('test-dir/without/with', 'without/file')) + self.assertResolve('test-dir/without/with/without/file', ('test-dir/without/with', 'without/file')) def test_Makefile(self): self.assertResolve('test-dir/with/Makefile', ('test-dir', 'with/Makefile')) self.assertResolve('test-dir/with/without/Makefile', ('test-dir/with', 'without/Makefile')) - self.assertResolve('test-dir/with/without/with/Makefile', - ('test-dir/with', 'without/with/Makefile')) + self.assertResolve('test-dir/with/without/with/Makefile', ('test-dir/with', 'without/with/Makefile')) self.assertResolve('test-dir/without/Makefile', ('test-dir', 'without/Makefile')) self.assertResolve('test-dir/without/with/Makefile', ('test-dir', 'without/with/Makefile')) - self.assertResolve('test-dir/without/with/without/Makefile', - ('test-dir/without/with', 'without/Makefile')) - + self.assertResolve('test-dir/without/with/without/Makefile', ('test-dir/without/with', 'without/Makefile')) class TestHierarchicalStringList(unittest.TestCase): def setUp(self): @@ -231,18 +224,18 @@ class TestHierarchicalStringList(unittest.TestCase): def test_exports_subdir(self): self.assertEqual(self.EXPORTS._children, {}) self.EXPORTS.foo += ["foo.h"] - self.assertItemsEqual(self.EXPORTS._children, {"foo": True}) + self.assertItemsEqual(self.EXPORTS._children, {"foo" : True}) self.assertEqual(self.EXPORTS.foo._strings, ["foo.h"]) self.EXPORTS.bar += ["bar.h"] self.assertItemsEqual(self.EXPORTS._children, - {"foo": True, "bar": True}) + {"foo" : True, "bar" : True}) self.assertEqual(self.EXPORTS.foo._strings, ["foo.h"]) self.assertEqual(self.EXPORTS.bar._strings, ["bar.h"]) def test_exports_multiple_subdir(self): self.EXPORTS.foo.bar = ["foobar.h"] - self.assertItemsEqual(self.EXPORTS._children, {"foo": True}) - self.assertItemsEqual(self.EXPORTS.foo._children, {"bar": True}) + self.assertItemsEqual(self.EXPORTS._children, {"foo" : True}) + self.assertItemsEqual(self.EXPORTS.foo._children, {"bar" : True}) self.assertItemsEqual(self.EXPORTS.foo.bar._children, {}) self.assertEqual(self.EXPORTS._strings, []) self.assertEqual(self.EXPORTS.foo._strings, []) @@ -277,24 +270,24 @@ class TestHierarchicalStringList(unittest.TestCase): "") def test_del_exports(self): - with self.assertRaises(MozbuildDeletionError): + with self.assertRaises(MozbuildDeletionError) as mde: self.EXPORTS.foo += ['bar.h'] del self.EXPORTS.foo def test_unsorted(self): - with self.assertRaises(UnsortedError): + with self.assertRaises(UnsortedError) as ee: self.EXPORTS += ['foo.h', 'bar.h'] - with self.assertRaises(UnsortedError): + with self.assertRaises(UnsortedError) as ee: self.EXPORTS.foo = ['foo.h', 'bar.h'] - with self.assertRaises(UnsortedError): + with self.assertRaises(UnsortedError) as ee: self.EXPORTS.foo += ['foo.h', 'bar.h'] def test_reassign(self): self.EXPORTS.foo = ['foo.h'] - with self.assertRaises(KeyError): + with self.assertRaises(KeyError) as ee: self.EXPORTS.foo = ['bar.h'] def test_walk(self): @@ -503,7 +496,7 @@ class TestStrictOrderingOnAppendListWithFlagsFactory(unittest.TestCase): l['a'] = 'foo' with self.assertRaises(Exception): - l['c'] + c = l['c'] self.assertEqual(l['a'].foo, False) l['a'].foo = True @@ -873,7 +866,6 @@ class TestMisc(unittest.TestCase): 'before abc between a b c after' ) - class TestEnumString(unittest.TestCase): def test_string(self): CompilerType = EnumString.subclass('gcc', 'clang', 'clang-cl') diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py index 3789b0fc2fad..2e5a6b6efafe 100644 --- a/python/mozbuild/mozbuild/testing.py +++ b/python/mozbuild/mozbuild/testing.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals import cPickle as pickle import os @@ -68,13 +68,11 @@ REFTEST_FLAVORS = ('crashtest', 'reftest') # Web platform tests have their own manifest format and are processed separately. WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',) - def all_test_flavors(): return ([v[0] for v in TEST_MANIFESTS.values()] + list(REFTEST_FLAVORS) + list(WEB_PLATFORM_TESTS_FLAVORS)) - class TestInstallInfo(object): def __init__(self): self.seen = set() @@ -90,7 +88,6 @@ class TestInstallInfo(object): self.deferred_installs |= other.deferred_installs return self - class SupportFilesConverter(object): """Processes a "support-files" entry from a test object, either from a parsed object from a test manifests or its representation in @@ -100,7 +97,6 @@ class SupportFilesConverter(object): effect, and the structure of the parsed objects from manifests will have a lot of repeated entries, so this class takes care of memoizing. """ - def __init__(self): self._fields = (('head', set()), ('support-files', set()), @@ -130,9 +126,8 @@ class SupportFilesConverter(object): # directory for the benefit of tests specifying 'install-to-subdir'. key = field, pattern, out_dir if key in info.seen: - raise ValueError( - "%s appears multiple times in a test manifest under a %s field," - " please omit the duplicate entry." % (pattern, field)) + raise ValueError("%s appears multiple times in a test manifest under a %s field," + " please omit the duplicate entry." % (pattern, field)) info.seen.add(key) if key in seen: continue @@ -180,7 +175,6 @@ class SupportFilesConverter(object): info.installs.append((full, mozpath.normpath(dest_path))) return info - def _resolve_installs(paths, topobjdir, manifest): """Using the given paths as keys, find any unresolved installs noted by the build backend corresponding to those keys, and add them @@ -194,9 +188,9 @@ def _resolve_installs(paths, topobjdir, manifest): path = path[2:] if path not in resolved_installs: raise Exception('A cross-directory support file path noted in a ' - 'test manifest does not appear in any other manifest.\n "%s" ' - 'must appear in another test manifest to specify an install ' - 'for "!/%s".' % (path, path)) + 'test manifest does not appear in any other manifest.\n "%s" ' + 'must appear in another test manifest to specify an install ' + 'for "!/%s".' % (path, path)) installs = resolved_installs[path] for install_info in installs: try: @@ -296,7 +290,6 @@ def read_manifestparser_manifest(context, manifest_path): finder=context._finder, handle_defaults=False) - def read_reftest_manifest(context, manifest_path): import reftest path = manifest_path.full_path @@ -304,7 +297,6 @@ def read_reftest_manifest(context, manifest_path): manifest.load(path) return manifest - def read_wpt_manifest(context, paths): manifest_path, tests_root = paths full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py index db454867da6c..d60b5729465c 100644 --- a/python/mozbuild/mozbuild/util.py +++ b/python/mozbuild/mozbuild/util.py @@ -23,6 +23,7 @@ import time import types from collections import ( + defaultdict, Iterable, OrderedDict, ) @@ -84,14 +85,12 @@ class EmptyValue(unicode): :py:class:`mozbuild.frontend.reader.EmptyConfig`. It should likely not be used elsewhere. """ - def __init__(self): super(EmptyValue, self).__init__() class ReadOnlyNamespace(object): """A class for objects with immutable attributes set at initialization.""" - def __init__(self, **kwargs): for k, v in kwargs.iteritems(): super(ReadOnlyNamespace, self).__setattr__(k, v) @@ -115,7 +114,6 @@ class ReadOnlyNamespace(object): class ReadOnlyDict(dict): """A read-only dictionary.""" - def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) @@ -138,7 +136,6 @@ undefined = undefined_default() class ReadOnlyDefaultDict(ReadOnlyDict): """A read-only dictionary that supports default values on retrieval.""" - def __init__(self, default_factory, *args, **kwargs): ReadOnlyDict.__init__(self, *args, **kwargs) self._default_factory = default_factory @@ -218,7 +215,6 @@ class FileAvoidWrite(BytesIO): out, but reports whether the file was existing and would have been updated still occur, as well as diff capture if requested. """ - def __init__(self, filename, capture_diff=False, dry_run=False, mode='rU'): BytesIO.__init__(self) self.name = filename @@ -296,7 +292,6 @@ class FileAvoidWrite(BytesIO): def __enter__(self): return self - def __exit__(self, type, value, traceback): if not self.closed: self.close() @@ -433,7 +428,7 @@ class UnsortedError(Exception): s.write('An attempt was made to add an unsorted sequence to a list. ') s.write('The incoming list is unsorted starting at element %d. ' % - self.i) + self.i) s.write('We expected "%s" but got "%s"' % ( self.sorted[self.i], self.original[self.i])) @@ -472,7 +467,7 @@ class StrictOrderingOnAppendListMixin(object): StrictOrderingOnAppendListMixin.ensure_sorted(sequence) return super(StrictOrderingOnAppendListMixin, self).__setslice__(i, j, - sequence) + sequence) def __add__(self, other): StrictOrderingOnAppendListMixin.ensure_sorted(other) @@ -486,18 +481,16 @@ class StrictOrderingOnAppendListMixin(object): class StrictOrderingOnAppendList(ListMixin, StrictOrderingOnAppendListMixin, - list): + list): """A list specialized for moz.build environments. We overload the assignment and append operations to require that incoming elements be ordered. This enforces cleaner style in moz.build files. """ - class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList): """Like StrictOrderingOnAppendList, but not allowing mutations of the value. """ - def append(self, elt): raise Exception("cannot use append on this type") @@ -516,7 +509,6 @@ class ImmutableStrictOrderingOnAppendList(StrictOrderingOnAppendList): class ListWithActionMixin(object): """Mixin to create lists with pre-processing. See ListWithAction.""" - def __init__(self, iterable=None, action=None): if iterable is None: iterable = [] @@ -540,9 +532,8 @@ class ListWithActionMixin(object): other = [self._action(i) for i in other] return super(ListWithActionMixin, self).__iadd__(other) - class StrictOrderingOnAppendListWithAction(StrictOrderingOnAppendListMixin, - ListMixin, ListWithActionMixin, list): + ListMixin, ListWithActionMixin, list): """An ordered list that accepts a callable to be applied to each item. A callable (action) passed to the constructor is run on each item of input. @@ -551,7 +542,6 @@ class StrictOrderingOnAppendListWithAction(StrictOrderingOnAppendListMixin, Note that the order of superclasses is therefore significant. """ - class ListWithAction(ListMixin, ListWithActionMixin, list): """A list that accepts a callable to be applied to each item. @@ -560,7 +550,6 @@ class ListWithAction(ListMixin, ListWithActionMixin, list): stored in place of the original input. """ - class MozbuildDeletionError(Exception): pass @@ -660,10 +649,8 @@ def StrictOrderingOnAppendListWithFlagsFactory(flags): (self._flags_type._flags, other._flags_type._flags)) intersection = set(self._flags.keys()) & set(other._flags.keys()) if intersection: - raise ValueError( - 'Cannot update flags: both lists of strings with flags configure %s' % - intersection - ) + raise ValueError('Cannot update flags: both lists of strings with flags configure %s' % + intersection) self._flags.update(other._flags) def extend(self, l): @@ -852,7 +839,7 @@ class LockFile(object): raise -def lock_file(lockfile, max_wait=600): +def lock_file(lockfile, max_wait = 600): """Create and hold a lockfile of the given name, with the given timeout. To release the lock, delete the returned object. @@ -880,8 +867,8 @@ def lock_file(lockfile, max_wait=600): s = os.stat(lockfile) except EnvironmentError as e: if e.errno == errno.ENOENT or e.errno == errno.EACCES: - # We didn't create the lockfile, so it did exist, but it's - # gone now. Just try again + # We didn't create the lockfile, so it did exist, but it's + # gone now. Just try again continue raise Exception('{0} exists but stat() failed: {1}'.format( @@ -893,7 +880,7 @@ def lock_file(lockfile, max_wait=600): if now - s[stat.ST_MTIME] > max_wait: pid = f.readline().rstrip() raise Exception('{0} has been locked for more than ' - '{1} seconds (PID {2})'.format(lockfile, max_wait, pid)) + '{1} seconds (PID {2})'.format(lockfile, max_wait, pid)) # It's not been locked too long, wait a while and retry f.close() @@ -910,7 +897,6 @@ def lock_file(lockfile, max_wait=600): class OrderedDefaultDict(OrderedDict): '''A combination of OrderedDict and defaultdict.''' - def __init__(self, default_factory, *args, **kwargs): OrderedDict.__init__(self, *args, **kwargs) self._default_factory = default_factory @@ -923,7 +909,6 @@ class OrderedDefaultDict(OrderedDict): class KeyedDefaultDict(dict): '''Like a defaultdict, but the default_factory function takes the key as argument''' - def __init__(self, default_factory, *args, **kwargs): dict.__init__(self, *args, **kwargs) self._default_factory = default_factory @@ -944,7 +929,6 @@ class memoize(dict): Both functions and instance methods are handled, although in the instance method case, the results are cache in the instance itself. ''' - def __init__(self, func): self.func = func functools.update_wrapper(self, func) @@ -972,7 +956,6 @@ class memoized_property(object): '''A specialized version of the memoize decorator that works for class instance properties. ''' - def __init__(self, func): self.func = func @@ -1024,7 +1007,7 @@ def TypedNamedTuple(name, fields): if not isinstance(value, ftype): raise TypeError('field in tuple not of proper type: %s; ' 'got %s, expected %s' % (fname, - type(value), ftype)) + type(value), ftype)) super(TypedTuple, self).__init__(*args, **kwargs) @@ -1058,7 +1041,7 @@ class TypedListMixin(object): sequence = self._ensure_type(sequence) return super(TypedListMixin, self).__setslice__(i, j, - sequence) + sequence) def __add__(self, other): other = self._ensure_type(other) @@ -1095,7 +1078,6 @@ def TypedList(type, base_class=List): return _TypedList - def group_unified_files(files, unified_prefix, unified_suffix, files_per_unified_file): """Return an iterator of (unified_filename, source_filenames) tuples. @@ -1119,7 +1101,6 @@ def group_unified_files(files, unified_prefix, unified_suffix, # don't want the fill value inserted by izip_longest to be an # issue. So we do a little dance to filter it out ourselves. dummy_fill_value = ("dummy",) - def filter_out_dummy(iterable): return itertools.ifilter(lambda x: x != dummy_fill_value, iterable) @@ -1171,7 +1152,6 @@ def expand_variables(s, variables): class DefinesAction(argparse.Action): '''An ArgumentParser action to handle -Dvar[=value] type of arguments.''' - def __call__(self, parser, namespace, values, option_string): defines = getattr(namespace, self.dest) if defines is None: @@ -1200,7 +1180,6 @@ class EnumString(unicode): subclasses. ''' POSSIBLE_VALUES = () - def __init__(self, value): if value not in self.POSSIBLE_VALUES: raise ValueError("'%s' is not a valid value for %s" @@ -1230,7 +1209,6 @@ def _escape_char(c): return "\\'" return unicode(c.encode('unicode_escape')) - # Mapping table between raw characters below \x80 and their escaped # counterpart, when they differ _INDENTED_REPR_TABLE = { @@ -1251,7 +1229,6 @@ def indented_repr(o, indent=4): assumes `from __future__ import unicode_literals`. ''' one_indent = ' ' * indent - def recurse_indented_repr(o, level): if isinstance(o, dict): yield '{\n' @@ -1352,7 +1329,6 @@ def patch_main(): import os import sys orig_find_module = imp.find_module - def my_find_module(name, dirs): if name == main_module_name: path = os.path.join(dirs[0], main_file_name) @@ -1362,7 +1338,6 @@ def patch_main(): # Don't allow writing bytecode file for the main module. orig_load_module = imp.load_module - def my_load_module(name, file, path, description): # multiprocess.forking invokes imp.load_module manually and # hard-codes the name __parents_main__ as the module name. @@ -1378,8 +1353,7 @@ def patch_main(): imp.find_module = my_find_module imp.load_module = my_load_module - from multiprocessing.forking import main - main() + from multiprocessing.forking import main; main() def my_get_command_line(): fork_code, lineno = inspect.getsourcelines(fork_interpose) diff --git a/python/mozbuild/mozbuild/vendor_aom.py b/python/mozbuild/mozbuild/vendor_aom.py index 31033550f12f..da0040266a5c 100644 --- a/python/mozbuild/mozbuild/vendor_aom.py +++ b/python/mozbuild/mozbuild/vendor_aom.py @@ -4,8 +4,10 @@ from __future__ import absolute_import, print_function, unicode_literals +from distutils.version import LooseVersion import logging from mozbuild.base import ( + BuildEnvironmentNotFoundException, MozbuildObject, ) import mozfile @@ -17,7 +19,6 @@ import sys import tarfile from urlparse import urlparse - class VendorAOM(MozbuildObject): def upstream_snapshot(self, revision): '''Construct a url for a tarball snapshot of the given revision.''' @@ -61,7 +62,7 @@ Please set a repository url with --repo on either googlesource or github.''' % h req.raise_for_status() try: info = req.json() - except ValueError: + except ValueError as e: # As of 2017 May, googlesource sends 4 garbage characters # at the beginning of the json response. Work around this. # https://bugs.chromium.org/p/chromium/issues/detail?id=718550 @@ -136,7 +137,7 @@ Please set a repository url with --repo on either googlesource or github.''' % h mozfile.remove(mozpath.join(target, '.gitattributes')) mozfile.remove(mozpath.join(target, '.gitignore')) mozfile.remove(mozpath.join(target, 'build', '.gitattributes')) - mozfile.remove(mozpath.join(target, 'build', '.gitignore')) + mozfile.remove(mozpath.join(target, 'build' ,'.gitignore')) def generate_sources(self, target): ''' diff --git a/python/mozbuild/mozbuild/vendor_dav1d.py b/python/mozbuild/mozbuild/vendor_dav1d.py index f61f87cb34e8..27b98c980658 100644 --- a/python/mozbuild/mozbuild/vendor_dav1d.py +++ b/python/mozbuild/mozbuild/vendor_dav1d.py @@ -4,8 +4,10 @@ from __future__ import absolute_import, print_function, unicode_literals +from distutils.version import LooseVersion import logging from mozbuild.base import ( + BuildEnvironmentNotFoundException, MozbuildObject, ) import mozfile @@ -17,7 +19,6 @@ import sys import tarfile from urlparse import urlparse - class VendorDav1d(MozbuildObject): def upstream_snapshot(self, revision): '''Construct a url for a tarball snapshot of the given revision.''' @@ -50,7 +51,7 @@ Please set a repository url with --repo on either googlesource or github.''' % h def upstream_gitlab_commit(self, revision): '''Query the github api for a git commit id and timestamp.''' - gitlab_api = 'https://code.videolan.org/api/v4/projects/videolan%2Fdav1d/repository/commits' # noqa + gitlab_api = 'https://code.videolan.org/api/v4/projects/videolan%2Fdav1d/repository/commits' url = mozpath.join(gitlab_api, revision) self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching commit id from {url}') @@ -95,8 +96,8 @@ Please set a repository url with --repo on either googlesource or github.''' % h prefix = ' release: commit' if prefix in yaml: new_yaml = re.sub(prefix + ' [v\.a-f0-9]+.*$', - prefix + ' %s (%s).' % (revision, timestamp), - yaml, flags=re.MULTILINE) + prefix + ' %s (%s).' % (revision, timestamp), + yaml, flags=re.MULTILINE) else: new_yaml = '%s\n\n%s %s.' % (yaml, prefix, revision) @@ -113,12 +114,13 @@ Please set a repository url with --repo on either googlesource or github.''' % h with open(dst_filename, 'w') as f: f.write(vcs_version) + def clean_upstream(self, target): '''Remove files we don't want to import.''' mozfile.remove(mozpath.join(target, '.gitattributes')) mozfile.remove(mozpath.join(target, '.gitignore')) mozfile.remove(mozpath.join(target, 'build', '.gitattributes')) - mozfile.remove(mozpath.join(target, 'build', '.gitignore')) + mozfile.remove(mozpath.join(target, 'build' ,'.gitignore')) def check_modified_files(self): ''' diff --git a/python/mozbuild/mozbuild/vendor_python.py b/python/mozbuild/mozbuild/vendor_python.py index 71bc0f221bf0..c5e8d00fb3dd 100644 --- a/python/mozbuild/mozbuild/vendor_python.py +++ b/python/mozbuild/mozbuild/vendor_python.py @@ -31,8 +31,7 @@ class VendorPython(MozbuildObject): self._activate_virtualenv() pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile') if not os.path.exists(pip_compile): - path = os.path.normpath(os.path.join( - self.topsrcdir, 'third_party', 'python', 'pip-tools')) + path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools')) self.virtualenv_manager.install_pip_package(path, vendored=True) spec = os.path.join(vendor_dir, 'requirements.in') requirements = os.path.join(vendor_dir, 'requirements.txt') diff --git a/python/mozbuild/mozbuild/vendor_rust.py b/python/mozbuild/mozbuild/vendor_rust.py index b9c22ca94064..24689c96b937 100644 --- a/python/mozbuild/mozbuild/vendor_rust.py +++ b/python/mozbuild/mozbuild/vendor_rust.py @@ -11,13 +11,13 @@ from mozbuild.base import ( BuildEnvironmentNotFoundException, MozbuildObject, ) +import mozfile import mozpack.path as mozpath import os import re import subprocess import sys - class VendorRust(MozbuildObject): def get_cargo_path(self): try: @@ -62,8 +62,7 @@ class VendorRust(MozbuildObject): on the user. Allow changes to Cargo.{toml,lock} since that's likely to be a common use case. ''' - modified = [f for f in self.repository.get_changed_files( - 'M') if os.path.basename(f) not in ('Cargo.toml', 'Cargo.lock')] + modified = [f for f in self.repository.get_changed_files('M') if os.path.basename(f) not in ('Cargo.toml', 'Cargo.lock')] if modified: self.log(logging.ERROR, 'modified_files', {}, '''You have uncommitted changes to the following files: @@ -109,31 +108,23 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- ''' cargo = self.get_cargo_path() if not self.check_cargo_version(cargo): - self.log(logging.ERROR, 'cargo_version', {}, - 'Cargo >= 0.13 required (install Rust 1.12 or newer)') + self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)') return None else: self.log(logging.DEBUG, 'cargo_version', {}, 'cargo is new enough') - have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output( - [cargo, '--list']).splitlines()) + have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines()) if not have_vendor: - self.log(logging.INFO, 'installing', {}, - 'Installing cargo-vendor (this may take a few minutes)...') + self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor (this may take a few minutes)...') env = self.check_openssl() self.run_process(args=[cargo, 'install', 'cargo-vendor'], append_env=env) elif not self.check_cargo_vendor_version(cargo): - self.log( - logging.INFO, 'cargo_vendor', {}, - ('cargo-vendor >= 0.1.23 required; ' - 'force-reinstalling (this may take a few minutes)...') - ) + self.log(logging.INFO, 'cargo_vendor', {}, 'cargo-vendor >= 0.1.23 required; force-reinstalling (this may take a few minutes)...') env = self.check_openssl() self.run_process(args=[cargo, 'install', '--force', 'cargo-vendor'], append_env=env) else: - self.log(logging.DEBUG, 'cargo_vendor', {}, - 'sufficiently new cargo-vendor is already installed') + self.log(logging.DEBUG, 'cargo_vendor', {}, 'sufficiently new cargo-vendor is already installed') return cargo @@ -239,7 +230,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- if not self.runtime_license(package, license): if license not in self.BUILDTIME_LICENSE_WHITELIST: self.log(logging.ERROR, 'package_license_error', {}, - '''Package {} has a non-approved license: {}. + '''Package {} has a non-approved license: {}. Please request license review on the package's license. If the package's license is approved, please add it to the whitelist of suitable licenses. @@ -247,7 +238,7 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- return False elif package not in self.BUILDTIME_LICENSE_WHITELIST[license]: self.log(logging.ERROR, 'package_license_error', {}, - '''Package {} has a license that is approved for build-time dependencies: {} + '''Package {} has a license that is approved for build-time dependencies: {} but the package itself is not whitelisted as being a build-time only package. If your package is build-time only, please add it to the whitelist of build-time @@ -267,10 +258,8 @@ Please commit or stash these changes before vendoring, or re-run with `--ignore- # to scanning individual lines. with open(toml_file, 'r') as f: license_lines = [l for l in f if l.strip().startswith(b'license')] - license_matches = list( - filter(lambda x: x, [LICENSE_LINE_RE.match(l) for l in license_lines])) - license_file_matches = list( - filter(lambda x: x, [LICENSE_FILE_LINE_RE.match(l) for l in license_lines])) + license_matches = list(filter(lambda x: x, [LICENSE_LINE_RE.match(l) for l in license_lines])) + license_file_matches = list(filter(lambda x: x, [LICENSE_FILE_LINE_RE.match(l) for l in license_lines])) # License information is optional for crates to provide, but # we require it. @@ -306,8 +295,7 @@ to the whitelist of packages whose licenses are suitable. return False approved_hash = self.RUNTIME_LICENSE_FILE_PACKAGE_WHITELIST[package] - license_contents = open(os.path.join( - vendor_dir, package, license_file), 'r').read() + license_contents = open(os.path.join(vendor_dir, package, license_file), 'r').read() current_hash = hashlib.sha256(license_contents).hexdigest() if current_hash != approved_hash: self.log(logging.ERROR, 'package_license_file_mismatch', {}, @@ -323,8 +311,7 @@ license file's hash. # Force all of the packages to be checked for license information # before reducing via `all`, so all license issues are found in a # single `mach vendor rust` invocation. - results = [check_package(p) for p in os.listdir(vendor_dir) - if os.path.isdir(os.path.join(vendor_dir, p))] + results = [check_package(p) for p in os.listdir(vendor_dir) if os.path.isdir(os.path.join(vendor_dir, p))] return all(results) def vendor(self, ignore_modified=False, @@ -342,17 +329,14 @@ license file's hash. vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir) # We use check_call instead of mozprocess to ensure errors are displayed. - # We do an |update -p| here to regenerate the Cargo.lock file with minimal - # changes. See bug 1324462 + # We do an |update -p| here to regenerate the Cargo.lock file with minimal changes. See bug 1324462 subprocess.check_call([cargo, 'update', '-p', 'gkrust'], cwd=self.topsrcdir) - subprocess.check_call([cargo, 'vendor', '--quiet', '--sync', - 'Cargo.lock'] + [vendor_dir], cwd=self.topsrcdir) + subprocess.check_call([cargo, 'vendor', '--quiet', '--sync', 'Cargo.lock'] + [vendor_dir], cwd=self.topsrcdir) if not self._check_licenses(vendor_dir): - self.log( - logging.ERROR, 'license_check_failed', {}, - '''The changes from `mach vendor rust` will NOT be added to version control.''') + self.log(logging.ERROR, 'license_check_failed', {}, + '''The changes from `mach vendor rust` will NOT be added to version control.''') sys.exit(1) self.repository.add_remove_files(vendor_dir) @@ -394,6 +378,4 @@ The changes from `mach vendor rust` will NOT be added to version control. Please consider finding ways to reduce the size of the vendored packages. For instance, check the vendored packages for unusually large test or benchmark files that don't need to be published to crates.io and submit -a pull request upstream to ignore those files when publishing.'''.format( - size=cumulative_added_size) - ) +a pull request upstream to ignore those files when publishing.'''.format(size=cumulative_added_size)) diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py index 183f221fd72f..17c7268de7aa 100644 --- a/python/mozbuild/mozbuild/virtualenv.py +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -47,14 +47,13 @@ class VirtualenvManager(object): """Contains logic for managing virtualenvs for building the tree.""" def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle, - manifest_path): + manifest_path): """Create a new manager. Each manager is associated with a source directory, a path where you want the virtualenv to be created, and a handle to write output to. """ - assert os.path.isabs( - manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path) + assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path) self.topsrcdir = topsrcdir self.topobjdir = topobjdir self.virtualenv_root = virtualenv_path @@ -72,7 +71,7 @@ class VirtualenvManager(object): def virtualenv_script_path(self): """Path to virtualenv's own populator script.""" return os.path.join(self.topsrcdir, 'third_party', 'python', - 'virtualenv', 'virtualenv.py') + 'virtualenv', 'virtualenv.py') @property def bin_path(self): @@ -128,7 +127,7 @@ class VirtualenvManager(object): # check if virtualenv exists if not os.path.exists(self.virtualenv_root) or \ - not os.path.exists(self.activate_path): + not os.path.exists(self.activate_path): return False @@ -199,12 +198,12 @@ class VirtualenvManager(object): env.pop('PYTHONDONTWRITEBYTECODE', None) args = [python, self.virtualenv_script_path, - # Without this, virtualenv.py may attempt to contact the outside - # world and search for or download a newer version of pip, - # setuptools, or wheel. This is bad for security, reproducibility, - # and speed. - '--no-download', - self.virtualenv_root] + # Without this, virtualenv.py may attempt to contact the outside + # world and search for or download a newer version of pip, + # setuptools, or wheel. This is bad for security, reproducibility, + # and speed. + '--no-download', + self.virtualenv_root] result = self._log_process_output(args, env=env) @@ -280,7 +279,7 @@ class VirtualenvManager(object): assert len(package) >= 2 self.call_setup(os.path.join(self.topsrcdir, package[1]), - package[2:]) + package[2:]) return True @@ -330,10 +329,10 @@ class VirtualenvManager(object): try: handle_package(package[1:]) return True - except Exception: - print('Error processing command. Ignoring', - 'because optional. (%s)' % ':'.join(package), - file=self.log_handle) + except: + print('Error processing command. Ignoring', \ + 'because optional. (%s)' % ':'.join(package), + file=self.log_handle) return False if package[0] in ('windows', '!windows'): @@ -377,7 +376,7 @@ class VirtualenvManager(object): # Python is smart enough to find a proper compiler and to use the # proper compiler flags. If it isn't your Python is likely broken. IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', - 'PYTHONDONTWRITEBYTECODE') + 'PYTHONDONTWRITEBYTECODE') try: old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None) @@ -406,13 +405,13 @@ class VirtualenvManager(object): # explained at # http://stackoverflow.com/questions/3047542/building-lxml-for-python-2-7-on-windows/5122521#5122521. if sys.platform in ('win32', 'cygwin') and \ - 'VS90COMNTOOLS' not in os.environ: + 'VS90COMNTOOLS' not in os.environ: warnings.warn('Hacking environment to allow binary Python ' - 'extensions to build. You can make this warning go away ' - 'by installing Visual Studio 2008. You can download the ' - 'Express Edition installer from ' - 'http://go.microsoft.com/?linkid=7729279') + 'extensions to build. You can make this warning go away ' + 'by installing Visual Studio 2008. You can download the ' + 'Express Edition installer from ' + 'http://go.microsoft.com/?linkid=7729279') # We list in order from oldest to newest to prefer the closest # to 2008 so differences are minimized. @@ -477,7 +476,7 @@ class VirtualenvManager(object): # the virtualenv for paths to be proper. args = [self.python_path, __file__, 'populate', self.topsrcdir, - self.topobjdir, self.virtualenv_root, self.manifest_path] + self.topobjdir, self.virtualenv_root, self.manifest_path] result = self._log_process_output(args, cwd=self.topsrcdir) @@ -659,9 +658,7 @@ def verify_python_version(log_handle): if __name__ == '__main__': if len(sys.argv) < 5: - print( - 'Usage: populate_virtualenv.py /path/to/topsrcdir ' - '/path/to/topobjdir /path/to/virtualenv /path/to/virtualenv_manifest') + print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/topobjdir /path/to/virtualenv /path/to/virtualenv_manifest') sys.exit(1) verify_python_version(sys.stdout) @@ -675,7 +672,7 @@ if __name__ == '__main__': topsrcdir, topobjdir, virtualenv_path, manifest_path = sys.argv[2:] manager = VirtualenvManager(topsrcdir, topobjdir, virtualenv_path, - sys.stdout, manifest_path) + sys.stdout, manifest_path) if populate: manager.populate() diff --git a/python/mozbuild/mozpack/archive.py b/python/mozbuild/mozpack/archive.py index 093f1a3ea058..0df33be7aebb 100644 --- a/python/mozbuild/mozpack/archive.py +++ b/python/mozbuild/mozpack/archive.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import bz2 import gzip diff --git a/python/mozbuild/mozpack/chrome/flags.py b/python/mozbuild/mozpack/chrome/flags.py index 0eaf587d0666..92e9ff4eca9f 100644 --- a/python/mozbuild/mozpack/chrome/flags.py +++ b/python/mozbuild/mozpack/chrome/flags.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import re from distutils.version import LooseVersion diff --git a/python/mozbuild/mozpack/chrome/manifest.py b/python/mozbuild/mozpack/chrome/manifest.py index 65fd7f418e23..b62a179495db 100644 --- a/python/mozbuild/mozpack/chrome/manifest.py +++ b/python/mozbuild/mozpack/chrome/manifest.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import re import os diff --git a/python/mozbuild/mozpack/copier.py b/python/mozbuild/mozpack/copier.py index 1a4df7dcf2c7..7f3bc4668e16 100644 --- a/python/mozbuild/mozpack/copier.py +++ b/python/mozbuild/mozpack/copier.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import stat diff --git a/python/mozbuild/mozpack/dmg.py b/python/mozbuild/mozpack/dmg.py index 59268a505ce0..165ae63f15d8 100644 --- a/python/mozbuild/mozpack/dmg.py +++ b/python/mozbuild/mozpack/dmg.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import buildconfig import errno import mozfile diff --git a/python/mozbuild/mozpack/errors.py b/python/mozbuild/mozpack/errors.py index 6c5892d3c65b..8b4b80072ba5 100644 --- a/python/mozbuild/mozpack/errors.py +++ b/python/mozbuild/mozpack/errors.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import sys from contextlib import contextmanager @@ -97,7 +97,7 @@ class ErrorCollector(object): if self._count is None: raise ErrorMessage(msg) self._count += 1 - print(msg, file=self.out) + print >>self.out, msg def fatal(self, msg): self._handle(self.FATAL, msg) diff --git a/python/mozbuild/mozpack/executables.py b/python/mozbuild/mozpack/executables.py index 27fd2f770741..c4c1e41059d1 100644 --- a/python/mozbuild/mozpack/executables.py +++ b/python/mozbuild/mozpack/executables.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import os import struct diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py index c8aaae2aadcb..aa540acf0986 100644 --- a/python/mozbuild/mozpack/files.py +++ b/python/mozbuild/mozpack/files.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import errno import inspect diff --git a/python/mozbuild/mozpack/hg.py b/python/mozbuild/mozpack/hg.py index 1cbdae14b426..79876061f4b1 100644 --- a/python/mozbuild/mozpack/hg.py +++ b/python/mozbuild/mozpack/hg.py @@ -27,7 +27,7 @@ # do not wish to do so, delete this exception statement from your # version. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import mercurial.error as error import mercurial.hg as hg diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py index 27eeb7c39f53..1bffcf1806f9 100644 --- a/python/mozbuild/mozpack/manifests.py +++ b/python/mozbuild/mozpack/manifests.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals from contextlib import contextmanager import json diff --git a/python/mozbuild/mozpack/mozjar.py b/python/mozbuild/mozpack/mozjar.py index 386a67a5a290..577c87d15fa3 100644 --- a/python/mozbuild/mozpack/mozjar.py +++ b/python/mozbuild/mozpack/mozjar.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import from io import BytesIO import struct diff --git a/python/mozbuild/mozpack/packager/__init__.py b/python/mozbuild/mozpack/packager/__init__.py index b52d71b0653e..d05817903be2 100644 --- a/python/mozbuild/mozpack/packager/__init__.py +++ b/python/mozbuild/mozpack/packager/__init__.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import from mozbuild.preprocessor import Preprocessor import re diff --git a/python/mozbuild/mozpack/packager/formats.py b/python/mozbuild/mozpack/packager/formats.py index ae5f66a15dcb..1e51ed8b812e 100644 --- a/python/mozbuild/mozpack/packager/formats.py +++ b/python/mozbuild/mozpack/packager/formats.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import from mozpack.chrome.manifest import ( Manifest, diff --git a/python/mozbuild/mozpack/packager/l10n.py b/python/mozbuild/mozpack/packager/l10n.py index 06808ddea516..a9db05e249a6 100644 --- a/python/mozbuild/mozpack/packager/l10n.py +++ b/python/mozbuild/mozpack/packager/l10n.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import ''' Replace localized parts of a packaged directory with data from a langpack diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py index 994cc5bc20ef..140056c09573 100644 --- a/python/mozbuild/mozpack/packager/unpack.py +++ b/python/mozbuild/mozpack/packager/unpack.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import mozpack.path as mozpath from mozpack.files import ( diff --git a/python/mozbuild/mozpack/path.py b/python/mozbuild/mozpack/path.py index a9828e7fffd2..19995cb981db 100644 --- a/python/mozbuild/mozpack/path.py +++ b/python/mozbuild/mozpack/path.py @@ -8,7 +8,7 @@ separators (always use forward slashes). Also contains a few additional utilities not found in :py:mod:`os.path`. ''' -from __future__ import absolute_import, print_function +from __future__ import absolute_import import ctypes import posixpath diff --git a/python/mozbuild/mozpack/test/support/minify_js_verify.py b/python/mozbuild/mozpack/test/support/minify_js_verify.py index 050e322604c4..8e4e8b7590b6 100644 --- a/python/mozbuild/mozpack/test/support/minify_js_verify.py +++ b/python/mozbuild/mozpack/test/support/minify_js_verify.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import print_function import sys diff --git a/python/mozbuild/mozpack/test/test_archive.py b/python/mozbuild/mozpack/test/test_archive.py index ec867485d6f9..1a9ef673da95 100644 --- a/python/mozbuild/mozpack/test/test_archive.py +++ b/python/mozbuild/mozpack/test/test_archive.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function +from __future__ import absolute_import import hashlib import os diff --git a/python/mozbuild/mozpack/test/test_chrome_flags.py b/python/mozbuild/mozpack/test/test_chrome_flags.py index 047e04399438..e6a5257e952f 100644 --- a/python/mozbuild/mozpack/test/test_chrome_flags.py +++ b/python/mozbuild/mozpack/test/test_chrome_flags.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest import mozunit from mozpack.chrome.flags import ( diff --git a/python/mozbuild/mozpack/test/test_chrome_manifest.py b/python/mozbuild/mozpack/test/test_chrome_manifest.py index 6fe00b0dc35d..67d84d889e37 100644 --- a/python/mozbuild/mozpack/test/test_chrome_manifest.py +++ b/python/mozbuild/mozpack/test/test_chrome_manifest.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest import mozunit import os diff --git a/python/mozbuild/mozpack/test/test_copier.py b/python/mozbuild/mozpack/test/test_copier.py index 5f75e8ee0f40..e44cf3df1c25 100644 --- a/python/mozbuild/mozpack/test/test_copier.py +++ b/python/mozbuild/mozpack/test/test_copier.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozpack.copier import ( FileCopier, FileRegistry, diff --git a/python/mozbuild/mozpack/test/test_errors.py b/python/mozbuild/mozpack/test/test_errors.py index 509260f02c81..a8d355ece522 100644 --- a/python/mozbuild/mozpack/test/test_errors.py +++ b/python/mozbuild/mozpack/test/test_errors.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozpack.errors import ( errors, ErrorMessage, diff --git a/python/mozbuild/mozpack/test/test_files.py b/python/mozbuild/mozpack/test/test_files.py index eb3f87c25aee..3103456e658b 100644 --- a/python/mozbuild/mozpack/test/test_files.py +++ b/python/mozbuild/mozpack/test/test_files.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozbuild.util import ensureParentDir from mozpack.errors import ( diff --git a/python/mozbuild/mozpack/test/test_manifests.py b/python/mozbuild/mozpack/test/test_manifests.py index becf455bda39..7f59bc823deb 100644 --- a/python/mozbuild/mozpack/test/test_manifests.py +++ b/python/mozbuild/mozpack/test/test_manifests.py @@ -2,7 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import unicode_literals import os diff --git a/python/mozbuild/mozpack/test/test_mozjar.py b/python/mozbuild/mozpack/test/test_mozjar.py index d9cd83436477..9187fc437cf1 100644 --- a/python/mozbuild/mozpack/test/test_mozjar.py +++ b/python/mozbuild/mozpack/test/test_mozjar.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozpack.files import FileFinder from mozpack.mozjar import ( JarReaderError, diff --git a/python/mozbuild/mozpack/test/test_packager.py b/python/mozbuild/mozpack/test/test_packager.py index 028f0cd0066a..457b8442cf9d 100644 --- a/python/mozbuild/mozpack/test/test_packager.py +++ b/python/mozbuild/mozpack/test/test_packager.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest import mozunit import os diff --git a/python/mozbuild/mozpack/test/test_packager_formats.py b/python/mozbuild/mozpack/test/test_packager_formats.py index afd30bda27b7..bda1ebd1a7bb 100644 --- a/python/mozbuild/mozpack/test/test_packager_formats.py +++ b/python/mozbuild/mozpack/test/test_packager_formats.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import mozunit import unittest from mozpack.packager.formats import ( diff --git a/python/mozbuild/mozpack/test/test_packager_l10n.py b/python/mozbuild/mozpack/test/test_packager_l10n.py index 2c5ff7a5c1eb..c797eadd176d 100644 --- a/python/mozbuild/mozpack/test/test_packager_l10n.py +++ b/python/mozbuild/mozpack/test/test_packager_l10n.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import unittest import mozunit from test_packager import MockFinder diff --git a/python/mozbuild/mozpack/test/test_packager_unpack.py b/python/mozbuild/mozpack/test/test_packager_unpack.py index d360c303eaac..350d0acef18a 100644 --- a/python/mozbuild/mozpack/test/test_packager_unpack.py +++ b/python/mozbuild/mozpack/test/test_packager_unpack.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - import mozunit from mozpack.packager.formats import ( FlatFormatter, diff --git a/python/mozbuild/mozpack/test/test_path.py b/python/mozbuild/mozpack/test/test_path.py index ec1394232785..ee41e4a6933f 100644 --- a/python/mozbuild/mozpack/test/test_path.py +++ b/python/mozbuild/mozpack/test/test_path.py @@ -2,8 +2,6 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import absolute_import, print_function - from mozpack.path import ( relpath, join,