From 26650da24dc7ec5ccead594f9a28516a9b34ca2a Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 26 Sep 2012 09:43:53 -0700 Subject: [PATCH] Bug 780329 - Part 5: Add base modules to mozbuild; r=glandium, jhammel --- python/mozbuild/mozbuild/base.py | 363 ++++++++++++++++++ .../locale/en-US/LC_MESSAGES/mozbuild.mo | Bin 0 -> 301 bytes .../locale/en-US/LC_MESSAGES/mozbuild.po | 8 + python/mozbuild/mozbuild/test/test_base.py | 61 +++ 4 files changed, 432 insertions(+) create mode 100644 python/mozbuild/mozbuild/base.py create mode 100644 python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo create mode 100644 python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po create mode 100644 python/mozbuild/mozbuild/test/test_base.py diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py new file mode 100644 index 000000000000..5d607d7fd080 --- /dev/null +++ b/python/mozbuild/mozbuild/base.py @@ -0,0 +1,363 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import logging +import multiprocessing +import os +import pymake.parser +import shlex +import subprocess +import which + +from mozprocess.processhandler import ProcessHandlerMixin +from pymake.data import Makefile +from tempfile import TemporaryFile + +from mozbuild.config import ConfigProvider +from mozbuild.config import PositiveIntegerType + + +# Perform detection of operating system environment. This is used by command +# execution. We only do this once to save redundancy. Yes, this can fail module +# loading. That is arguably OK. +if 'SHELL' in os.environ: + _current_shell = os.environ['SHELL'] +elif 'MOZILLABUILD' in os.environ: + _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' +elif 'COMSPEC' in os.environ: + _current_shell = os.environ['COMSPEC'] +else: + raise Exception('Could not detect environment shell!') + +_in_msys = False + +if os.environ.get('MSYSTEM', None) == 'MINGW32': + _in_msys = True + + if not _current_shell.lower().endswith('.exe'): + _current_shell += '.exe' + + +class MozbuildObject(object): + """Base class providing basic functionality useful to many modules. + + Modules in this package typically require common functionality such as + accessing the current config, getting the location of the source directory, + running processes, etc. This classes provides that functionality. Other + modules can inherit from this class to obtain this functionality easily. + """ + def __init__(self, topsrcdir, settings, log_manager, topobjdir=None): + """Create a new Mozbuild object instance. + + Instances are bound to a source directory, a ConfigSettings instance, + and a LogManager instance. The topobjdir may be passed in as well. If + it isn't, it will be calculated from the active mozconfig. + """ + self.topsrcdir = topsrcdir + self.settings = settings + self.config = BuildConfig(settings) + self.logger = logging.getLogger(__name__) + self.log_manager = log_manager + + self._config_guess_output = None + self._make = None + self._topobjdir = topobjdir + + @property + def topobjdir(self): + if self._topobjdir is None: + self._load_mozconfig() + + if self._topobjdir is None: + self._topobjdir = 'obj-%s' % self._config_guess + + return self._topobjdir + + @property + def distdir(self): + return os.path.join(self.topobjdir, 'dist') + + @property + def bindir(self): + return os.path.join(self.topobjdir, 'dist', 'bin') + + @property + def statedir(self): + return os.path.join(self.topobjdir, '.mozbuild') + + def log(self, level, action, params, format_str): + self.logger.log(level, format_str, + extra={'action': action, 'params': params}) + + def _load_mozconfig(self, path=None): + # The mozconfig loader outputs a make file. We parse and load this make + # file with pymake and evaluate it in a context similar to client.mk. + + loader = os.path.join(self.topsrcdir, 'build', 'autoconf', + 'mozconfig2client-mk') + + # os.environ from a library function is somewhat evil. But, mozconfig + # files are tightly coupled with the environment by definition. In the + # future, perhaps we'll have a more sanitized environment for mozconfig + # execution. + env = dict(os.environ) + if path is not None: + env['MOZCONFIG'] = path + + env['CONFIG_GUESS'] = self._config_guess + + output = subprocess.check_output([loader, self.topsrcdir], + stderr=subprocess.PIPE, cwd=self.topsrcdir, env=env) + + # The output is make syntax. We parse this in a specialized make + # context. + statements = pymake.parser.parsestring(output, 'mozconfig') + + makefile = Makefile(workdir=self.topsrcdir, env={ + 'TOPSRCDIR': self.topsrcdir, + 'CONFIG_GUESS': self._config_guess}) + + statements.execute(makefile) + + def get_value(name): + exp = makefile.variables.get(name)[2] + + return exp.resolvestr(makefile, makefile.variables) + + for name, flavor, source, value in makefile.variables: + # We only care about variables that came from the parsed mozconfig. + if source != pymake.data.Variables.SOURCE_MAKEFILE: + continue + + # Ignore some pymake built-ins. + if name in ('.PYMAKE', 'MAKELEVEL', 'MAKEFLAGS'): + continue + + if name == 'MOZ_OBJDIR': + self._topobjdir = get_value(name) + + # If we want to extract other variables defined by mozconfig, here + # is where we'd do it. + + @property + def _config_guess(self): + if self._config_guess_output is None: + p = os.path.join(self.topsrcdir, 'build', 'autoconf', + 'config.guess') + self._config_guess_output = subprocess.check_output([p], + cwd=self.topsrcdir).strip() + + return self._config_guess_output + + def _ensure_objdir_exists(self): + if os.path.isdir(self.statedir): + return + + os.makedirs(self.statedir) + + def _ensure_state_subdir_exists(self, subdir): + path = os.path.join(self.statedir, subdir) + + if os.path.isdir(path): + return + + os.makedirs(path) + + def _get_state_filename(self, filename, subdir=None): + path = self.statedir + + if subdir: + path = os.path.join(path, subdir) + + return os.path.join(path, filename) + + def _get_srcdir_path(self, path): + """Convert a relative path in the source directory to a full path.""" + return os.path.join(self.topsrcdir, path) + + def _get_objdir_path(self, path): + """Convert a relative path in the object directory to a full path.""" + return os.path.join(self.topobjdir, path) + + def _run_make(self, directory=None, filename=None, target=None, log=True, + srcdir=False, allow_parallel=True, line_handler=None, env=None, + ignore_errors=False): + """Invoke make. + + directory -- Relative directory to look for Makefile in. + filename -- Explicit makefile to run. + target -- Makefile target(s) to make. Can be a string or iterable of + strings. + srcdir -- If True, invoke make from the source directory tree. + Otherwise, make will be invoked from the object directory. + """ + self._ensure_objdir_exists() + + args = [self._make_path] + + if directory: + args.extend(['-C', directory]) + + if filename: + args.extend(['-f', filename]) + + if allow_parallel: + args.append('-j%d' % self.settings.build.threads) + + if ignore_errors: + args.append('-k') + + # Silent mode by default. + args.append('-s') + + # Print entering/leaving directory messages. Some consumers look at + # these to measure progress. Ideally, we'd do everything with pymake + # and use hooks in its API. Unfortunately, it doesn't provide that + # feature... yet. + args.append('-w') + + if isinstance(target, list): + args.extend(target) + elif target: + args.append(target) + + fn = self._run_command_in_objdir + + if srcdir: + fn = self._run_command_in_srcdir + + params = { + 'args': args, + 'line_handler': line_handler, + 'explicit_env': env, + 'log_level': logging.INFO, + 'require_unix_environment': True, + 'ignore_errors': ignore_errors, + } + + if log: + params['log_name'] = 'make' + + fn(**params) + + @property + def _make_path(self): + if self._make is None: + if self._is_windows(): + self._make = os.path.join(self.topsrcdir, 'build', 'pymake', + 'make.py') + + else: + for test in ['gmake', 'make']: + try: + self._make = which.which(test) + break + except which.WhichError: + continue + + if self._make is None: + raise Exception('Could not find suitable make binary!') + + return self._make + + def _run_command_in_srcdir(self, **args): + self._run_command(cwd=self.topsrcdir, **args) + + def _run_command_in_objdir(self, **args): + self._run_command(cwd=self.topobjdir, **args) + + def _run_command(self, args=None, cwd=None, append_env=None, + explicit_env=None, log_name=None, log_level=logging.INFO, + line_handler=None, require_unix_environment=False, + ignore_errors=False): + """Runs a single command to completion. + + Takes a list of arguments to run where the first item is the + executable. Runs the command in the specified directory and + with optional environment variables. + + append_env -- Dict of environment variables to append to the current + set of environment variables. + explicit_env -- Dict of environment variables to set for the new + process. Any existing environment variables will be ignored. + + require_unix_environment if True will ensure the command is executed + within a UNIX environment. Basically, if we are on Windows, it will + execute the command via an appropriate UNIX-like shell. + """ + assert isinstance(args, list) and len(args) + + if require_unix_environment and _in_msys: + # Always munge Windows-style into Unix style for the command. + prog = args[0].replace('\\', '/') + + # PyMake removes the C: prefix. But, things seem to work here + # without it. Not sure what that's about. + + # We run everything through the msys shell. We need to use + # '-c' and pass all the arguments as one argument because that is + # how sh works. + cline = subprocess.list2cmdline([prog] + args[1:]) + args = [_current_shell, '-c', cline] + + self.log(logging.INFO, 'process', {'args': args}, ' '.join(args)) + + def handleLine(line): + if line_handler: + line_handler(line) + + if not log_name: + return + + self.log(log_level, log_name, {'line': line.strip()}, '{line}') + + use_env = {} + if explicit_env: + use_env = explicit_env + else: + use_env.update(os.environ) + + if append_env: + use_env.update(env) + + p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, + processOutputLine=[handleLine], universal_newlines=True) + p.run() + p.processOutput() + status = p.wait() + + if status != 0 and not ignore_errors: + raise Exception('Process executed with non-0 exit code: %s' % args) + + def _is_windows(self): + return os.name in ('nt', 'ce') + + def _spawn(self, cls): + """Create a new MozbuildObject-derived class instance from ourselves. + + This is used as a convenience method to create other + MozbuildObject-derived class instances. It can only be used on + classes that have the same constructor arguments as us. + """ + + return cls(self.topsrcdir, self.settings, self.log_manager, + topobjdir=self.topobjdir) + + +class BuildConfig(ConfigProvider): + """The configuration for mozbuild.""" + + def __init__(self, settings): + self.settings = settings + + @classmethod + def _register_settings(cls): + def register(section, option, type_cls, **kwargs): + cls.register_setting(section, option, type_cls, domain='mozbuild', + **kwargs) + + register('build', 'threads', PositiveIntegerType, + default=multiprocessing.cpu_count()) diff --git a/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo new file mode 100644 index 0000000000000000000000000000000000000000..be7711cb2fcfc927de59407bac54fb613dfab0fe GIT binary patch literal 301 zcmZ9{%S{9^3NFmrd))^IJEx+du>vOZ|go(%>V!Z literal 0 HcmV?d00001 diff --git a/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po new file mode 100644 index 000000000000..fbdfabd83043 --- /dev/null +++ b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po @@ -0,0 +1,8 @@ +msgid "build.threads.short" +msgstr "Thread Count" + +msgid "build.threads.full" +msgstr "The number of threads to use when performing CPU intensive tasks. " + "This constrols the level of parallelization. The default value is " + "the number of cores in your machine." + diff --git a/python/mozbuild/mozbuild/test/test_base.py b/python/mozbuild/mozbuild/test/test_base.py new file mode 100644 index 000000000000..5206492a98ab --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_base.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import unittest + +from tempfile import NamedTemporaryFile + +from mozbuild.base import BuildConfig +from mozbuild.base import MozbuildObject +from mozbuild.config import ConfigSettings +from mozbuild.logger import LoggingManager + + +curdir = os.path.dirname(__file__) +topsrcdir = os.path.normpath(os.path.join(curdir, '..', '..', '..', '..')) +log_manager = LoggingManager() + + +class TestBuildConfig(unittest.TestCase): + def test_basic(self): + c = ConfigSettings() + c.register_provider(BuildConfig) + + c.build.threads = 6 + + +class TestMozbuildObject(unittest.TestCase): + def get_base(self): + settings = ConfigSettings() + settings.register_provider(BuildConfig) + + return MozbuildObject(topsrcdir, settings, log_manager) + + def test_mozconfig_parsing(self): + with NamedTemporaryFile(mode='wt') as mozconfig: + mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir') + mozconfig.flush() + + os.environ['MOZCONFIG'] = mozconfig.name + + base = self.get_base() + base._load_mozconfig() + + self.assertEqual(base.topobjdir, '%s/some-objdir' % topsrcdir) + + del os.environ['MOZCONFIG'] + + def test_objdir_config_guess(self): + base = self.get_base() + + with NamedTemporaryFile() as mozconfig: + os.environ['MOZCONFIG'] = mozconfig.name + + self.assertIsNotNone(base.topobjdir) + self.assertEqual(len(base.topobjdir.split()), 1) + + del os.environ['MOZCONFIG']