diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py index cc8e74be800a..12b2a27c458b 100644 --- a/python/mozbuild/mozbuild/backend/common.py +++ b/python/mozbuild/mozbuild/backend/common.py @@ -4,6 +4,7 @@ from __future__ import absolute_import, unicode_literals +import cPickle as pickle import itertools import json import os @@ -370,19 +371,18 @@ class CommonBackend(BuildBackend): # Write out a machine-readable file describing every test. topobjdir = self.environment.topobjdir - with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh: - json.dump(self._test_manager.tests_by_path, fh) + with self._write_file(mozpath.join(topobjdir, 'all-tests.pkl'), mode='rb') as fh: + pickle.dump(dict(self._test_manager.tests_by_path), fh, protocol=2) - with self._write_file(mozpath.join(topobjdir, 'test-defaults.json')) as fh: - json.dump(self._test_manager.manifest_defaults, fh) + with self._write_file(mozpath.join(topobjdir, 'test-defaults.pkl'), mode='rb') as fh: + pickle.dump(self._test_manager.manifest_defaults, fh, protocol=2) - path = mozpath.join(self.environment.topobjdir, 'test-installs.json') - with self._write_file(path) as fh: - json.dump({k: v for k, v in self._test_manager.installs_by_path.items() - if k in self._test_manager.deferred_installs}, - fh, - sort_keys=True, - indent=4) + path = mozpath.join(self.environment.topobjdir, 'test-installs.pkl') + with self._write_file(path, mode='rb') as fh: + pickle.dump({k: v for k, v in self._test_manager.installs_by_path.items() + if k in self._test_manager.deferred_installs}, + fh, + protocol=2) # Write out a machine-readable file describing binaries. with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh: diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py index df4fa958d3eb..87f50f497379 100644 --- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals +import cPickle as pickle import json import os import unittest @@ -524,11 +525,11 @@ class TestRecursiveMakeBackend(BackendTester): '[include:xpcshell.ini]', ]) - all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json') + all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl') self.assertTrue(os.path.exists(all_tests_path)) - with open(all_tests_path, 'rt') as fh: - o = json.load(fh) + with open(all_tests_path, 'rb') as fh: + o = pickle.load(fh) self.assertIn('xpcshell.js', o) self.assertIn('dir1/test_bar.js', o) @@ -550,12 +551,12 @@ class TestRecursiveMakeBackend(BackendTester): def test_test_manifest_deffered_installs_written(self): """Shared support files are written to their own data file by the backend.""" env = self._consume('test-manifest-shared-support', RecursiveMakeBackend) - all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json') + all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl') self.assertTrue(os.path.exists(all_tests_path)) - test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json') + test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl') with open(test_installs_path, 'r') as fh: - test_installs = json.load(fh) + test_installs = pickle.load(fh) self.assertEqual(set(test_installs.keys()), set(['child/test_sub.js', @@ -572,7 +573,7 @@ class TestRecursiveMakeBackend(BackendTester): # First, read the generated for ini manifest contents. m = InstallManifest(path=test_files_manifest) - # Then, synthesize one from the test-installs.json file. This should + # Then, synthesize one from the test-installs.pkl file. This should # allow us to re-create a subset of the above. synthesized_manifest = InstallManifest() for item, installs in test_installs.items(): @@ -850,11 +851,11 @@ class TestRecursiveMakeBackend(BackendTester): """Ensure test suites honor package_tests=False.""" env = self._consume('test-manifests-package-tests', RecursiveMakeBackend) - all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json') + all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl') self.assertTrue(os.path.exists(all_tests_path)) - with open(all_tests_path, 'rt') as fh: - o = json.load(fh) + with open(all_tests_path, 'rb') as fh: + o = pickle.load(fh) self.assertIn('mochitest.js', o) self.assertIn('not_packaged.java', o) diff --git a/python/mozbuild/mozbuild/test/test_testing.py b/python/mozbuild/mozbuild/test/test_testing.py index 853109413482..e71892e241ae 100644 --- a/python/mozbuild/mozbuild/test/test_testing.py +++ b/python/mozbuild/mozbuild/test/test_testing.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals +import cPickle as pickle import os import shutil import tempfile @@ -21,8 +22,7 @@ from mozbuild.testing import ( ) -ALL_TESTS_JSON = b''' -{ +ALL_TESTS = { "accessible/tests/mochitest/actions/test_anchors.html": [ { "dir_relpath": "accessible/tests/mochitest/actions", @@ -154,11 +154,11 @@ ALL_TESTS_JSON = b''' "tags": "devtools" } ] -}'''.strip() +} -TEST_DEFAULTS = b'''{ - "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\\ndata/**\\nxpcshell_updater.ini"} -}''' +TEST_DEFAULTS = { + "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"} +} class Base(unittest.TestCase): @@ -172,13 +172,13 @@ class Base(unittest.TestCase): self._temp_files = [] def _get_test_metadata(self): - all_tests = NamedTemporaryFile() - all_tests.write(ALL_TESTS_JSON) + all_tests = NamedTemporaryFile(mode='wb') + pickle.dump(ALL_TESTS, all_tests) all_tests.flush() self._temp_files.append(all_tests) - test_defaults = NamedTemporaryFile() - test_defaults.write(TEST_DEFAULTS) + test_defaults = NamedTemporaryFile(mode='wb') + pickle.dump(TEST_DEFAULTS, test_defaults) test_defaults.flush() self._temp_files.append(test_defaults) @@ -251,10 +251,10 @@ class TestTestResolver(Base): topobjdir = tempfile.mkdtemp() self._temp_dirs.append(topobjdir) - with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh: - fh.write(ALL_TESTS_JSON) - with open(os.path.join(topobjdir, 'test-defaults.json'), 'wt') as fh: - fh.write(TEST_DEFAULTS) + with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh: + pickle.dump(ALL_TESTS, fh) + with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh: + pickle.dump(TEST_DEFAULTS, fh) o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py index ac891d4b9c98..3409f98219c5 100644 --- a/python/mozbuild/mozbuild/testing.py +++ b/python/mozbuild/mozbuild/testing.py @@ -4,7 +4,7 @@ from __future__ import absolute_import, unicode_literals -import json +import cPickle as pickle import os import sys @@ -56,12 +56,12 @@ class TestMetadata(object): self._tests_by_flavor = defaultdict(set) self._test_dirs = set() - with open(all_tests, 'rt') as fh: - test_data = json.load(fh) + with open(all_tests, 'rb') as fh: + test_data = pickle.load(fh) defaults = None if test_defaults: - with open(test_defaults, 'rt') as fh: - defaults = json.load(fh) + with open(test_defaults, 'rb') as fh: + defaults = pickle.load(fh) for path, tests in test_data.items(): for metadata in tests: if defaults: @@ -178,14 +178,14 @@ class TestResolver(MozbuildObject): # If installing tests is going to result in re-generating the build # backend, we need to do this here, so that the updated contents of - # all-tests.json make it to the set of tests to run. + # all-tests.pkl make it to the set of tests to run. self._run_make(target='run-tests-deps', pass_thru=True, print_directory=False) self._tests = TestMetadata(os.path.join(self.topobjdir, - 'all-tests.json'), + 'all-tests.pkl'), test_defaults=os.path.join(self.topobjdir, - 'test-defaults.json')) + 'test-defaults.pkl')) self._test_rewrites = { 'a11y': os.path.join(self.topobjdir, '_tests', 'testing', @@ -417,9 +417,9 @@ def _resolve_installs(paths, topobjdir, manifest): by the build backend corresponding to those keys, and add them to the given manifest. """ - filename = os.path.join(topobjdir, 'test-installs.json') - with open(filename, 'r') as fh: - resolved_installs = json.load(fh) + filename = os.path.join(topobjdir, 'test-installs.pkl') + with open(filename, 'rb') as fh: + resolved_installs = pickle.load(fh) for path in paths: path = path[2:]