Bug 1188224 - Remove stale .pyc files from the source directory at import time. r=gps

build/mach_bootstrap.py

@@ -9,6 +9,9 @@ import os
 import platform
 import sys
 import time
+import __builtin__
+
+from types import ModuleType
 
 
 STATE_DIR_FIRST_RUN = '''
@@ -327,3 +330,70 @@ def bootstrap(topsrcdir, mozilla_dir=None):
         mach.load_commands_from_file(os.path.join(mozilla_dir, path))
 
     return mach
+
+
+# Hook import such that .pyc/.pyo files without a corresponding .py file in
+# the source directory are essentially ignored. See further below for details
+# and caveats.
+# Objdirs outside the source directory are ignored because in most cases, if
+# a .pyc/.pyo file exists there, a .py file will be next to it anyways.
+class ImportHook(object):
+    def __init__(self, original_import):
+        self._original_import = original_import
+        # Assume the source directory is the parent directory of the one
+        # containing this file.
+        self._source_dir = os.path.normcase(os.path.abspath(
+            os.path.dirname(os.path.dirname(__file__)))) + os.sep
+        self._modules = set()
+
+    def __call__(self, name, globals=None, locals=None, fromlist=None,
+                 level=-1):
+        # name might be a relative import. Instead of figuring out what that
+        # resolves to, which is complex, just rely on the real import.
+        # Since we don't know the full module name, we can't check sys.modules,
+        # so we need to keep track of which modules we've already seen to avoid
+        # to stat() them again when they are imported multiple times.
+        module = self._original_import(name, globals, locals, fromlist, level)
+
+        # Some tests replace modules in sys.modules with non-module instances.
+        if not isinstance(module, ModuleType):
+            return module
+
+        resolved_name = module.__name__
+        if resolved_name in self._modules:
+            return module
+        self._modules.add(resolved_name)
+
+        # Builtin modules don't have a __file__ attribute.
+        if not hasattr(module, '__file__'):
+            return module
+
+        # Note: module.__file__ is not always absolute.
+        path = os.path.normcase(os.path.abspath(module.__file__))
+        # Note: we could avoid normcase and abspath above for non pyc/pyo
+        # files, but those are actually rare, so it doesn't really matter.
+        if not path.endswith(('.pyc', '.pyo')):
+            return module
+
+        # Ignore modules outside our source directory
+        if not path.startswith(self._source_dir):
+            return module
+
+        # If there is no .py corresponding to the .pyc/.pyo module we're
+        # loading, remove the .pyc/.pyo file, and reload the module.
+        # Since we already loaded the .pyc/.pyo module, if it had side
+        # effects, they will have happened already, and loading the module
+        # with the same name, from another directory may have the same side
+        # effects (or different ones). We assume it's not a problem for the
+        # python modules under our source directory (either because it
+        # doesn't happen or because it doesn't matter).
+        if not os.path.exists(module.__file__[:-1]):
+            os.remove(module.__file__)
+            del sys.modules[module.__name__]
+            module = self(name, globals, locals, fromlist, level)
+
+        return module
+
+
+# Install our hook
+__builtin__.__import__ = ImportHook(__builtin__.__import__)

python/mozbuild/mozbuild/virtualenv.py

@@ -194,9 +194,9 @@ class VirtualenvManager(object):
         """
 
         packages = self.packages()
+        python_lib = distutils.sysconfig.get_python_lib()
 
         def handle_package(package):
-            python_lib = distutils.sysconfig.get_python_lib()
             if package[0] == 'setup.py':
                 assert len(package) >= 2
 
@@ -331,6 +331,16 @@ class VirtualenvManager(object):
 
             for package in packages:
                 handle_package(package)
+
+            sitecustomize = os.path.join(
+                os.path.dirname(os.__file__), 'sitecustomize.py')
+            with open(sitecustomize, 'w') as f:
+                f.write(
+                    '# Importing mach_bootstrap has the side effect of\n'
+                    '# installing an import hook\n'
+                    'import mach_bootstrap\n'
+                )
+
         finally:
             os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)