Bug 1168607 - Use mozpack Finders for reading moz.build files; r=glandium

Sometimes moz.build data may not come from the local filesystem. To
support defining moz.build data in other backing stores, we switch
moz.build reading to use mozpack's *Finder classes for I/O.

The use of a FileFinder bound to / is sub-optimal. We should ideally
be creating a finder bound to topsrcdir. However, doing this would
require refactoring a lot of path handling in the reader, as that code
makes many assumptions that paths are absolute. This would be excellent
follow-up work.

While I was here, I cleaned up some linter warnings for unused imports.

On my machine, this commit results in a slight slowdown of moz.build
reading. Before, `mach build-backend` was consistently reporting 1.99s
for reading 2,572 moz.build files. After, it is consistently reporting
2.07s, an increase of 4%. This is likely due to a) function call
overhead b) the cost of instantiating a few thousand File instances
c) FileFinder performing an os.path.exists() before returning File
instances. I believe the regression is tolerable.

--HG--
extra : commitid : 1WDcrSU3SQD
extra : rebase_source : a23aa5a4063c6f7080ee00b6f0fe0ee532da3ce7
This commit is contained in:
Gregory Szorc 2015-06-09 18:16:35 -07:00
Родитель 4bee702035
Коммит 8ce366a5f7
2 изменённых файлов: 20 добавлений и 17 удалений

Просмотреть файл

@ -39,7 +39,6 @@ from mozbuild.util import (
EmptyValue,
memoize,
ReadOnlyDefaultDict,
ReadOnlyDict,
)
from mozbuild.backend.configenvironment import ConfigEnvironment
@ -53,6 +52,7 @@ from .data import (
)
from .sandbox import (
default_finder,
SandboxError,
SandboxExecutionError,
SandboxLoadError,
@ -172,10 +172,10 @@ class MozbuildSandbox(Sandbox):
metadata is a dict of metadata that can be used during the sandbox
evaluation.
"""
def __init__(self, context, metadata={}):
def __init__(self, context, metadata={}, finder=default_finder):
assert isinstance(context, Context)
Sandbox.__init__(self, context)
Sandbox.__init__(self, context, finder=finder)
self._log = logging.getLogger(__name__)
@ -391,7 +391,8 @@ class MozbuildSandbox(Sandbox):
'special_variables': self.metadata.get('special_variables', {}),
'subcontexts': self.metadata.get('subcontexts', {}),
'templates': self.metadata.get('templates', {})
})
}, finder=self._finder)
template.exec_in_sandbox(sandbox, *args, **kwargs)
# This is gross, but allows the merge to happen. Eventually, the
@ -853,12 +854,13 @@ class BuildReader(object):
each sandbox evaluation. Its return value is ignored.
"""
def __init__(self, config):
def __init__(self, config, finder=default_finder):
self.config = config
self._log = logging.getLogger(__name__)
self._read_files = set()
self._execution_stack = []
self._finder = finder
def read_topsrcdir(self):
"""Read the tree of linked moz.build files.
@ -1091,7 +1093,8 @@ class BuildReader(object):
config.external_source_dir = None
context = Context(VARIABLES, config)
sandbox = MozbuildSandbox(context, metadata=metadata)
sandbox = MozbuildSandbox(context, metadata=metadata,
finder=self._finder)
sandbox.exec_file(path)
context.execution_time = time.time() - time_start
@ -1127,7 +1130,7 @@ class BuildReader(object):
non_unified_sources = set()
for s in gyp_dir.non_unified_sources:
source = SourcePath(context, s)
if not os.path.exists(source.full_path):
if not self._finder.get(source.full_path):
raise SandboxValidationError('Cannot find %s.' % source,
context)
non_unified_sources.add(source)
@ -1215,7 +1218,7 @@ class BuildReader(object):
@memoize
def exists(path):
return os.path.exists(path)
return self._finder.get(path) is not None
def itermozbuild(path):
subpath = ''

Просмотреть файл

@ -19,15 +19,16 @@ user-friendly error messages in the case of errors.
from __future__ import unicode_literals
import copy
import os
import sys
import weakref
from contextlib import contextmanager
from mozbuild.util import ReadOnlyDict
from context import Context
from .context import Context
from mozpack.files import FileFinder
default_finder = FileFinder('/', find_executables=False)
def alphabetical_sorted(iterable, cmp=None, key=lambda x: x.lower(),
@ -108,7 +109,7 @@ class Sandbox(dict):
'int': int,
})
def __init__(self, context, builtins=None):
def __init__(self, context, builtins=None, finder=default_finder):
"""Initialize a Sandbox ready for execution.
"""
self._builtins = builtins or self.BUILTINS
@ -132,6 +133,8 @@ class Sandbox(dict):
# Current literal source being executed.
self._current_source = None
self._finder = finder
@property
def _context(self):
return self._active_contexts[-1]
@ -143,11 +146,8 @@ class Sandbox(dict):
"""
assert os.path.isabs(path)
source = None
try:
with open(path, 'rt') as fd:
source = fd.read()
source = self._finder.get(path).read()
except Exception as e:
raise SandboxLoadError(self._context.source_stack,
sys.exc_info()[2], read_error=path)