Bug 1312574 - Use cPickle instead of json to serialize the build system's tests database. r=gps

MozReview-Commit-ID: 9xmUtVV6SRN

--HG--
extra : rebase_source : 957bfacfa5c13fa88b298e33dcaf475c08be8776

python/mozbuild/mozbuild/backend/common.py

@@ -4,6 +4,7 @@
 
 from __future__ import absolute_import, unicode_literals
 
+import cPickle as pickle
 import itertools
 import json
 import os
@@ -370,19 +371,18 @@ class CommonBackend(BuildBackend):
 
         # Write out a machine-readable file describing every test.
         topobjdir = self.environment.topobjdir
-        with self._write_file(mozpath.join(topobjdir, 'all-tests.json')) as fh:
-            json.dump(self._test_manager.tests_by_path, fh)
+        with self._write_file(mozpath.join(topobjdir, 'all-tests.pkl'), mode='rb') as fh:
+            pickle.dump(dict(self._test_manager.tests_by_path), fh, protocol=2)
 
-        with self._write_file(mozpath.join(topobjdir, 'test-defaults.json')) as fh:
-            json.dump(self._test_manager.manifest_defaults, fh)
+        with self._write_file(mozpath.join(topobjdir, 'test-defaults.pkl'), mode='rb') as fh:
+            pickle.dump(self._test_manager.manifest_defaults, fh, protocol=2)
 
-        path = mozpath.join(self.environment.topobjdir, 'test-installs.json')
-        with self._write_file(path) as fh:
-            json.dump({k: v for k, v in self._test_manager.installs_by_path.items()
-                       if k in self._test_manager.deferred_installs},
-                      fh,
-                      sort_keys=True,
-                      indent=4)
+        path = mozpath.join(self.environment.topobjdir, 'test-installs.pkl')
+        with self._write_file(path, mode='rb') as fh:
+            pickle.dump({k: v for k, v in self._test_manager.installs_by_path.items()
+                         if k in self._test_manager.deferred_installs},
+                        fh,
+                        protocol=2)
 
         # Write out a machine-readable file describing binaries.
         with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -4,6 +4,7 @@
 
 from __future__ import unicode_literals
 
+import cPickle as pickle
 import json
 import os
 import unittest
@@ -524,11 +525,11 @@ class TestRecursiveMakeBackend(BackendTester):
             '[include:xpcshell.ini]',
         ])
 
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
         self.assertTrue(os.path.exists(all_tests_path))
 
-        with open(all_tests_path, 'rt') as fh:
-            o = json.load(fh)
+        with open(all_tests_path, 'rb') as fh:
+            o = pickle.load(fh)
 
             self.assertIn('xpcshell.js', o)
             self.assertIn('dir1/test_bar.js', o)
@@ -550,12 +551,12 @@ class TestRecursiveMakeBackend(BackendTester):
     def test_test_manifest_deffered_installs_written(self):
         """Shared support files are written to their own data file by the backend."""
         env = self._consume('test-manifest-shared-support', RecursiveMakeBackend)
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
         self.assertTrue(os.path.exists(all_tests_path))
-        test_installs_path = mozpath.join(env.topobjdir, 'test-installs.json')
+        test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl')
 
         with open(test_installs_path, 'r') as fh:
-            test_installs = json.load(fh)
+            test_installs = pickle.load(fh)
 
         self.assertEqual(set(test_installs.keys()),
                          set(['child/test_sub.js',
@@ -572,7 +573,7 @@ class TestRecursiveMakeBackend(BackendTester):
         # First, read the generated for ini manifest contents.
         m = InstallManifest(path=test_files_manifest)
 
-        # Then, synthesize one from the test-installs.json file. This should
+        # Then, synthesize one from the test-installs.pkl file. This should
         # allow us to re-create a subset of the above.
         synthesized_manifest = InstallManifest()
         for item, installs in test_installs.items():
@@ -850,11 +851,11 @@ class TestRecursiveMakeBackend(BackendTester):
         """Ensure test suites honor package_tests=False."""
         env = self._consume('test-manifests-package-tests', RecursiveMakeBackend)
 
-        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.json')
+        all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl')
         self.assertTrue(os.path.exists(all_tests_path))
 
-        with open(all_tests_path, 'rt') as fh:
-            o = json.load(fh)
+        with open(all_tests_path, 'rb') as fh:
+            o = pickle.load(fh)
             self.assertIn('mochitest.js', o)
             self.assertIn('not_packaged.java', o)
 

python/mozbuild/mozbuild/test/test_testing.py

@@ -4,6 +4,7 @@
 
 from __future__ import unicode_literals
 
+import cPickle as pickle
 import os
 import shutil
 import tempfile
@@ -21,8 +22,7 @@ from mozbuild.testing import (
 )
 
 
-ALL_TESTS_JSON = b'''
-{
+ALL_TESTS = {
     "accessible/tests/mochitest/actions/test_anchors.html": [
         {
             "dir_relpath": "accessible/tests/mochitest/actions",
@@ -154,11 +154,11 @@ ALL_TESTS_JSON = b'''
             "tags": "devtools"
         }
    ]
-}'''.strip()
+}
 
-TEST_DEFAULTS = b'''{
-    "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\\ndata/**\\nxpcshell_updater.ini"}
-}'''
+TEST_DEFAULTS = {
+    "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"}
+}
 
 
 class Base(unittest.TestCase):
@@ -172,13 +172,13 @@ class Base(unittest.TestCase):
         self._temp_files = []
 
     def _get_test_metadata(self):
-        all_tests = NamedTemporaryFile()
-        all_tests.write(ALL_TESTS_JSON)
+        all_tests = NamedTemporaryFile(mode='wb')
+        pickle.dump(ALL_TESTS, all_tests)
         all_tests.flush()
         self._temp_files.append(all_tests)
 
-        test_defaults = NamedTemporaryFile()
-        test_defaults.write(TEST_DEFAULTS)
+        test_defaults = NamedTemporaryFile(mode='wb')
+        pickle.dump(TEST_DEFAULTS, test_defaults)
         test_defaults.flush()
         self._temp_files.append(test_defaults)
 
@@ -251,10 +251,10 @@ class TestTestResolver(Base):
         topobjdir = tempfile.mkdtemp()
         self._temp_dirs.append(topobjdir)
 
-        with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh:
-            fh.write(ALL_TESTS_JSON)
-        with open(os.path.join(topobjdir, 'test-defaults.json'), 'wt') as fh:
-            fh.write(TEST_DEFAULTS)
+        with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh:
+            pickle.dump(ALL_TESTS, fh)
+        with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh:
+            pickle.dump(TEST_DEFAULTS, fh)
 
         o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir)
 

python/mozbuild/mozbuild/testing.py

@@ -4,7 +4,7 @@
 
 from __future__ import absolute_import, unicode_literals
 
-import json
+import cPickle as pickle
 import os
 import sys
 
@@ -56,12 +56,12 @@ class TestMetadata(object):
         self._tests_by_flavor = defaultdict(set)
         self._test_dirs = set()
 
-        with open(all_tests, 'rt') as fh:
-            test_data = json.load(fh)
+        with open(all_tests, 'rb') as fh:
+            test_data = pickle.load(fh)
         defaults = None
         if test_defaults:
-            with open(test_defaults, 'rt') as fh:
-                defaults = json.load(fh)
+            with open(test_defaults, 'rb') as fh:
+                defaults = pickle.load(fh)
         for path, tests in test_data.items():
             for metadata in tests:
                 if defaults:
@@ -178,14 +178,14 @@ class TestResolver(MozbuildObject):
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
-        # all-tests.json make it to the set of tests to run.
+        # all-tests.pkl make it to the set of tests to run.
         self._run_make(target='run-tests-deps', pass_thru=True,
                        print_directory=False)
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
-                                                'all-tests.json'),
+                                                'all-tests.pkl'),
                                    test_defaults=os.path.join(self.topobjdir,
-                                                              'test-defaults.json'))
+                                                              'test-defaults.pkl'))
 
         self._test_rewrites = {
             'a11y': os.path.join(self.topobjdir, '_tests', 'testing',
@@ -417,9 +417,9 @@ def _resolve_installs(paths, topobjdir, manifest):
     by the build backend corresponding to those keys, and add them
     to the given manifest.
     """
-    filename = os.path.join(topobjdir, 'test-installs.json')
-    with open(filename, 'r') as fh:
-        resolved_installs = json.load(fh)
+    filename = os.path.join(topobjdir, 'test-installs.pkl')
+    with open(filename, 'rb') as fh:
+        resolved_installs = pickle.load(fh)
 
     for path in paths:
         path = path[2:]