зеркало из https://github.com/mozilla/gecko-dev.git
Bug 802210 - Refactor virtualenv integration; r=glandium
We now populate the virtualenv at the beginning of configure. We have also refactored how the virtualenv is populated. populate_virtualenv.py is completely refactored. Its default action now takes the topsrcdir and virtualenv paths and ensures a virtualenv is created, populated, and up-to-date. If it is out of date, it repopulates it. populate_virtualenv.py also now performs the Python version check validation instead of configure. It's easier to manage in Python than to have configure do it.
This commit is contained in:
Gregory Szorc
Родитель
ad50b9e765
Коммит
74c7681f33
|
|
@ -1,110 +1,315 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
# You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
# This file contains code for populating the virtualenv environment for
|
||||
# Mozilla's build system. It is typically called as part of configure.
|
||||
|
||||
from __future__ import with_statement
|
||||
from __future__ import print_function, unicode_literals, with_statement
|
||||
|
||||
import distutils.sysconfig
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import distutils.sysconfig
|
||||
|
||||
def populate_virtualenv(top_source_directory, manifest_filename, log_handle):
|
||||
"""Populate the virtualenv from the contents of a manifest.
|
||||
|
||||
The manifest file consists of colon-delimited fields. The first field
|
||||
specifies the action. The remaining fields are arguments to that action.
|
||||
The following actions are supported:
|
||||
# Minimum version of Python required to build.
|
||||
MINIMUM_PYTHON_MAJOR = 2
|
||||
MINIMUM_PYTHON_MINOR = 5
|
||||
|
||||
setup.py -- Invoke setup.py for a package. Expects the arguments:
|
||||
1. relative path directory containing setup.py.
|
||||
2. argument(s) to setup.py. e.g. "develop". Each program argument is
|
||||
delimited by a colon. Arguments with colons are not yet supported.
|
||||
|
||||
filename.pth -- Adds the path given as argument to filename.pth under
|
||||
the virtualenv site packages directory.
|
||||
class VirtualenvManager(object):
|
||||
"""Contains logic for managing virtualenvs for building the tree."""
|
||||
|
||||
optional -- This denotes the action as optional. The requested action
|
||||
is attempted. If it fails, we issue a warning and go on. The initial
|
||||
"optional" field is stripped then the remaining line is processed
|
||||
like normal. e.g. "optional:setup.py:python/foo:built_ext:-i"
|
||||
def __init__(self, topsrcdir, virtualenv_path, log_handle):
|
||||
"""Create a new manager.
|
||||
|
||||
copy -- Copies the given file in the virtualenv site packages directory.
|
||||
Each manager is associated with a source directory, a path where you
|
||||
want the virtualenv to be created, and a handle to write output to.
|
||||
"""
|
||||
self.topsrcdir = topsrcdir
|
||||
self.virtualenv_root = virtualenv_path
|
||||
self.log_handle = log_handle
|
||||
|
||||
Note that the Python interpreter running this function should be the one
|
||||
from the virtualenv. If it is the system Python or if the environment is
|
||||
not configured properly, packages could be installed into the wrong place.
|
||||
This is how virtualenv's work.
|
||||
"""
|
||||
packages = []
|
||||
fh = open(manifest_filename, 'rU')
|
||||
for line in fh:
|
||||
packages.append(line.rstrip().split(':'))
|
||||
fh.close()
|
||||
@property
|
||||
def virtualenv_script_path(self):
|
||||
"""Path to virtualenv's own populator script."""
|
||||
return os.path.join(self.topsrcdir, 'python', 'virtualenv',
|
||||
'virtualenv.py')
|
||||
|
||||
def handle_package(package):
|
||||
python_lib = distutils.sysconfig.get_python_lib()
|
||||
if package[0] == 'setup.py':
|
||||
assert len(package) >= 2
|
||||
@property
|
||||
def manifest_path(self):
|
||||
return os.path.join(self.topsrcdir, 'build', 'virtualenv',
|
||||
'packages.txt')
|
||||
|
||||
call_setup(os.path.join(top_source_directory, package[1]),
|
||||
package[2:])
|
||||
@property
|
||||
def python_path(self):
|
||||
if sys.platform in ('win32', 'cygwin'):
|
||||
return os.path.join(self.virtualenv_root, 'Scripts', 'python.exe')
|
||||
|
||||
return True
|
||||
return os.path.join(self.virtualenv_root, 'bin', 'python')
|
||||
|
||||
if package[0] == 'copy':
|
||||
assert len(package) == 2
|
||||
@property
|
||||
def activate_path(self):
|
||||
if sys.platform in ('win32', 'cygwin'):
|
||||
return os.path.join(self.virtualenv_root, 'Scripts',
|
||||
'activate_this.py')
|
||||
|
||||
shutil.copy(os.path.join(top_source_directory, package[1]),
|
||||
os.path.join(python_lib, os.path.basename(package[1])))
|
||||
return os.path.join(self.virtualenv_root, 'bin', 'activate_this.py')
|
||||
|
||||
return True
|
||||
def ensure(self):
|
||||
"""Ensure the virtualenv is present and up to date.
|
||||
|
||||
if package[0].endswith('.pth'):
|
||||
assert len(package) == 2
|
||||
If the virtualenv is up to date, this does nothing. Otherwise, it
|
||||
creates and populates the virtualenv as necessary.
|
||||
|
||||
with open(os.path.join(python_lib, package[0]), 'a') as f:
|
||||
f.write("%s\n" % os.path.join(top_source_directory, package[1]))
|
||||
This should be the main API used from this class as it is the
|
||||
highest-level.
|
||||
"""
|
||||
deps = [self.manifest_path, __file__]
|
||||
|
||||
return True
|
||||
if not os.path.exists(self.virtualenv_root) or \
|
||||
not os.path.exists(self.activate_path):
|
||||
|
||||
return self.build()
|
||||
|
||||
activate_mtime = os.path.getmtime(self.activate_path)
|
||||
dep_mtime = max(os.path.getmtime(p) for p in deps)
|
||||
|
||||
if dep_mtime > activate_mtime:
|
||||
return self.build()
|
||||
|
||||
return self.virtualenv_root
|
||||
|
||||
def create(self):
|
||||
"""Create a new, empty virtualenv.
|
||||
|
||||
Receives the path to virtualenv's virtualenv.py script (which will be
|
||||
called out to), the path to create the virtualenv in, and a handle to
|
||||
write output to.
|
||||
"""
|
||||
args = [sys.executable, self.virtualenv_script_path,
|
||||
'--system-site-packages', self.virtualenv_root]
|
||||
|
||||
result = subprocess.call(args, stdout=self.log_handle,
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
if result != 0:
|
||||
raise Exception('Error creating virtualenv.')
|
||||
|
||||
return self.virtualenv_root
|
||||
|
||||
def populate(self):
|
||||
"""Populate the virtualenv.
|
||||
|
||||
The manifest file consists of colon-delimited fields. The first field
|
||||
specifies the action. The remaining fields are arguments to that
|
||||
action. The following actions are supported:
|
||||
|
||||
setup.py -- Invoke setup.py for a package. Expects the arguments:
|
||||
1. relative path directory containing setup.py.
|
||||
2. argument(s) to setup.py. e.g. "develop". Each program argument
|
||||
is delimited by a colon. Arguments with colons are not yet
|
||||
supported.
|
||||
|
||||
filename.pth -- Adds the path given as argument to filename.pth under
|
||||
the virtualenv site packages directory.
|
||||
|
||||
optional -- This denotes the action as optional. The requested action
|
||||
is attempted. If it fails, we issue a warning and go on. The
|
||||
initial "optional" field is stripped then the remaining line is
|
||||
processed like normal. e.g.
|
||||
"optional:setup.py:python/foo:built_ext:-i"
|
||||
|
||||
copy -- Copies the given file in the virtualenv site packages
|
||||
directory.
|
||||
|
||||
Note that the Python interpreter running this function should be the
|
||||
one from the virtualenv. If it is the system Python or if the
|
||||
environment is not configured properly, packages could be installed
|
||||
into the wrong place. This is how virtualenv's work.
|
||||
"""
|
||||
packages = []
|
||||
fh = open(self.manifest_path, 'rUt')
|
||||
for line in fh:
|
||||
packages.append(line.rstrip().split(':'))
|
||||
fh.close()
|
||||
|
||||
def handle_package(package):
|
||||
python_lib = distutils.sysconfig.get_python_lib()
|
||||
if package[0] == 'setup.py':
|
||||
assert len(package) >= 2
|
||||
|
||||
self.call_setup(os.path.join(self.topsrcdir, package[1]),
|
||||
package[2:])
|
||||
|
||||
if package[0] == 'optional':
|
||||
try:
|
||||
handle_package(package[1:])
|
||||
return True
|
||||
except:
|
||||
print >>log_handle, 'Error processing command. Ignoring', \
|
||||
'because optional. (%s)' % ':'.join(package)
|
||||
return False
|
||||
|
||||
raise Exception('Unknown action: %s' % package[0])
|
||||
if package[0] == 'copy':
|
||||
assert len(package) == 2
|
||||
|
||||
for package in packages:
|
||||
handle_package(package)
|
||||
src = os.path.join(self.topsrcdir, package[1])
|
||||
dst = os.path.join(python_lib, os.path.basename(package[1]))
|
||||
|
||||
shutil.copy(src, dst)
|
||||
|
||||
return True
|
||||
|
||||
if package[0].endswith('.pth'):
|
||||
assert len(package) == 2
|
||||
|
||||
path = os.path.join(self.topsrcdir, package[1])
|
||||
|
||||
with open(os.path.join(python_lib, package[0]), 'a') as f:
|
||||
f.write("%s\n" % path)
|
||||
|
||||
return True
|
||||
|
||||
if package[0] == 'optional':
|
||||
try:
|
||||
handle_package(package[1:])
|
||||
return True
|
||||
except:
|
||||
print('Error processing command. Ignoring', \
|
||||
'because optional. (%s)' % ':'.join(package),
|
||||
file=self.log_handle)
|
||||
return False
|
||||
|
||||
raise Exception('Unknown action: %s' % package[0])
|
||||
|
||||
# We always target the OS X deployment target that Python itself was
|
||||
# built with, regardless of what's in the current environment. If we
|
||||
# don't do # this, we may run into a Python bug. See
|
||||
# http://bugs.python.org/issue9516 and bug 659881.
|
||||
#
|
||||
# Note that this assumes that nothing compiled in the virtualenv is
|
||||
# shipped as part of a distribution. If we do ship anything, the
|
||||
# deployment target here may be different from what's targeted by the
|
||||
# shipping binaries and # virtualenv-produced binaries may fail to
|
||||
# work.
|
||||
#
|
||||
# We also ignore environment variables that may have been altered by
|
||||
# configure or a mozconfig activated in the current shell. We trust
|
||||
# Python is smart enough to find a proper compiler and to use the
|
||||
# proper compiler flags. If it isn't your Python is likely broken.
|
||||
IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS')
|
||||
|
||||
try:
|
||||
old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
|
||||
sysconfig_target = \
|
||||
distutils.sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
|
||||
|
||||
if sysconfig_target is not None:
|
||||
os.environ['MACOSX_DEPLOYMENT_TARGET'] = sysconfig_target
|
||||
|
||||
old_env_variables = {}
|
||||
for k in IGNORE_ENV_VARIABLES:
|
||||
if k not in os.environ:
|
||||
continue
|
||||
|
||||
old_env_variables[k] = os.environ[k]
|
||||
del os.environ[k]
|
||||
|
||||
for package in packages:
|
||||
handle_package(package)
|
||||
finally:
|
||||
try:
|
||||
del os.environ['MACOSX_DEPLOYMENT_TARGET']
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if old_target is not None:
|
||||
os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
|
||||
|
||||
for k in old_env_variables:
|
||||
os.environ[k] = old_env_variables[k]
|
||||
|
||||
|
||||
def call_setup(directory, arguments):
|
||||
"""Calls setup.py in a directory."""
|
||||
setup = os.path.join(directory, 'setup.py')
|
||||
def call_setup(self, directory, arguments):
|
||||
"""Calls setup.py in a directory."""
|
||||
setup = os.path.join(directory, 'setup.py')
|
||||
|
||||
program = [sys.executable, setup]
|
||||
program.extend(arguments)
|
||||
program = [sys.executable, setup]
|
||||
program.extend(arguments)
|
||||
|
||||
# We probably could call the contents of this file inside the context of
|
||||
# this interpreter using execfile() or similar. However, if global
|
||||
# variables like sys.path are adjusted, this could cause all kinds of
|
||||
# havoc. While this may work, invoking a new process is safer.
|
||||
result = subprocess.call(program, cwd=directory)
|
||||
# We probably could call the contents of this file inside the context
|
||||
# of # this interpreter using execfile() or similar. However, if global
|
||||
# variables like sys.path are adjusted, this could cause all kinds of
|
||||
# havoc. While this may work, invoking a new process is safer.
|
||||
result = subprocess.call(program, cwd=directory)
|
||||
|
||||
if result != 0:
|
||||
raise Exception('Error installing package: %s' % directory)
|
||||
|
||||
def build(self):
|
||||
"""Build a virtualenv per tree conventions.
|
||||
|
||||
This returns the path of the created virtualenv.
|
||||
"""
|
||||
|
||||
self.create()
|
||||
|
||||
# We need to populate the virtualenv using the Python executable in
|
||||
# the virtualenv for paths to be proper.
|
||||
|
||||
args = [self.python_path, __file__, 'populate', self.topsrcdir,
|
||||
self.virtualenv_root]
|
||||
|
||||
result = subprocess.call(args, stdout=self.log_handle,
|
||||
stderr=subprocess.STDOUT, cwd=self.topsrcdir)
|
||||
|
||||
if result != 0:
|
||||
raise Exception('Error populating virtualenv.')
|
||||
|
||||
os.utime(self.activate_path, None)
|
||||
|
||||
return self.virtualenv_root
|
||||
|
||||
def activate(self):
|
||||
"""Activate the virtualenv in this Python context.
|
||||
|
||||
If you run a random Python script and wish to "activate" the
|
||||
virtualenv, you can simply instantiate an instance of this class
|
||||
and call .ensure() and .activate() to make the virtualenv active.
|
||||
"""
|
||||
|
||||
execfile(self.activate_path, dict(__file__=self.activate_path))
|
||||
|
||||
|
||||
def verify_python_version(log_handle):
|
||||
"""Ensure the current version of Python is sufficient."""
|
||||
major, minor = sys.version_info[:2]
|
||||
|
||||
if major != MINIMUM_PYTHON_MAJOR or minor < MINIMUM_PYTHON_MINOR:
|
||||
log_handle.write('Python %d.%d or greater (but not Python 3) is '
|
||||
'required to build. ' %
|
||||
(MINIMUM_PYTHON_MAJOR, MINIMUM_PYTHON_MINOR))
|
||||
log_handle.write('You are running Python %d.%d.\n' % (major, minor))
|
||||
sys.exit(1)
|
||||
|
||||
if result != 0:
|
||||
raise Exception('Error installing package: %s' % directory)
|
||||
|
||||
# configure invokes us with /path/to/topsrcdir and /path/to/manifest
|
||||
if __name__ == '__main__':
|
||||
assert len(sys.argv) == 3
|
||||
if len(sys.argv) < 3:
|
||||
print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/virtualenv')
|
||||
sys.exit(1)
|
||||
|
||||
verify_python_version(sys.stdout)
|
||||
|
||||
topsrcdir = sys.argv[1]
|
||||
virtualenv_path = sys.argv[2]
|
||||
populate = False
|
||||
|
||||
# This should only be called internally.
|
||||
if sys.argv[1] == 'populate':
|
||||
populate = True
|
||||
topsrcdir = sys.argv[2]
|
||||
virtualenv_path = sys.argv[3]
|
||||
|
||||
manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout)
|
||||
|
||||
if populate:
|
||||
manager.populate()
|
||||
else:
|
||||
manager.ensure()
|
||||
|
||||
populate_virtualenv(sys.argv[1], sys.argv[2], sys.stdout)
|
||||
sys.exit(0)
|
||||
|
|
|
|||
94
configure.in
94
configure.in
|
|
@ -62,8 +62,6 @@ dnl Set the minimum version of toolkit libs used by mozilla
|
|||
dnl ========================================================
|
||||
GLIB_VERSION=1.2.0
|
||||
PERL_VERSION=5.006
|
||||
PYTHON_VERSION_MAJOR=2
|
||||
PYTHON_VERSION_MINOR=5
|
||||
CAIRO_VERSION=1.10
|
||||
PANGO_VERSION=1.14.0
|
||||
GTK2_VERSION=2.10.0
|
||||
|
|
@ -125,6 +123,30 @@ then
|
|||
fi
|
||||
MOZ_BUILD_ROOT=`pwd`
|
||||
|
||||
MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python2.6 python2.5 python)
|
||||
if test -z "$PYTHON"; then
|
||||
AC_MSG_ERROR([python was not found in \$PATH])
|
||||
fi
|
||||
|
||||
AC_MSG_RESULT([Creating Python environment])
|
||||
dnl This verifies our Python version is sane and ensures the Python
|
||||
dnl virtualenv is present and up to date. It sanitizes the environment
|
||||
dnl for us, so we don't need to clean anything out.
|
||||
$PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
|
||||
$_topsrcdir $MOZ_BUILD_ROOT/_virtualenv || exit 1
|
||||
|
||||
dnl Create a virtualenv where we can install local Python packages
|
||||
case "$host_os" in
|
||||
mingw*)
|
||||
PYTHON=`cd $MOZ_BUILD_ROOT && pwd -W`/_virtualenv/Scripts/python.exe
|
||||
;;
|
||||
*)
|
||||
PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
|
||||
;;
|
||||
esac
|
||||
|
||||
AC_SUBST(PYTHON)
|
||||
|
||||
MOZ_DEFAULT_COMPILER
|
||||
|
||||
COMPILE_ENVIRONMENT=1
|
||||
|
|
@ -803,11 +825,6 @@ else
|
|||
AC_MSG_RESULT([yes])
|
||||
fi
|
||||
|
||||
MOZ_PATH_PROGS(PYTHON, $PYTHON python2.7 python2.6 python2.5 python)
|
||||
if test -z "$PYTHON"; then
|
||||
AC_MSG_ERROR([python was not found in \$PATH])
|
||||
fi
|
||||
|
||||
MOZ_ARG_WITH_BOOL(system-ply,
|
||||
[ --with-system-ply Use system installed python ply library],
|
||||
[if $PYTHON -c 'import ply' 2>&5; then
|
||||
|
|
@ -1839,24 +1856,6 @@ case "$host" in
|
|||
;;
|
||||
esac
|
||||
|
||||
dnl We require version 2.5 or newer of Python to build.
|
||||
AC_MSG_CHECKING([for Python version >= $PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR but not 3.x])
|
||||
|
||||
changequote(:)
|
||||
read python_version_major python_version_minor python_version_micro <<EOF
|
||||
`$PYTHON -c 'import sys; print sys.version_info[0], sys.version_info[1], sys.version_info[2]'`
|
||||
EOF
|
||||
changequote([,])
|
||||
|
||||
if test $python_version_major -ne $PYTHON_VERSION_MAJOR; then
|
||||
AC_MSG_ERROR([Cannot build on Python $python_version_major.])
|
||||
else
|
||||
if test $python_version_minor -lt $PYTHON_VERSION_MINOR; then
|
||||
AC_MSG_ERROR([Cannot build on Python $python_version_major.$python_version_minor. You need at least $PYTHON_VERSION_MAJOR.$PYTHON_VERSION_MINOR.])
|
||||
fi
|
||||
fi
|
||||
AC_MSG_RESULT([yes])
|
||||
|
||||
dnl Check for using a custom <stdint.h> implementation
|
||||
dnl ========================================================
|
||||
AC_MSG_CHECKING(for custom <stdint.h> implementation)
|
||||
|
|
@ -8823,51 +8822,6 @@ case "$host" in
|
|||
;;
|
||||
esac
|
||||
|
||||
dnl Create a virtualenv where we can install local Python packages
|
||||
AC_MSG_RESULT([Creating Python virtualenv])
|
||||
rm -rf _virtualenv
|
||||
mkdir -p _virtualenv
|
||||
MACOSX_DEPLOYMENT_TARGET= PYTHONDONTWRITEBYTECODE= $PYTHON $_topsrcdir/python/virtualenv/virtualenv.py --system-site-packages ./_virtualenv
|
||||
case "$host_os" in
|
||||
mingw*)
|
||||
PYTHON=$MOZ_BUILD_ROOT/_virtualenv/Scripts/python.exe
|
||||
;;
|
||||
*)
|
||||
PYTHON=$MOZ_BUILD_ROOT/_virtualenv/bin/python
|
||||
;;
|
||||
esac
|
||||
|
||||
AC_SUBST(PYTHON)
|
||||
|
||||
dnl Populate the virtualenv
|
||||
|
||||
dnl We always use the version Python was compiled with to ensure that compiled
|
||||
dnl extensions work properly. This works around a bug in Python. See also
|
||||
dnl http://bugs.python.org/issue9516 and bug 659881.
|
||||
dnl
|
||||
dnl Please note that this assumes nothing built during virtualenv population is
|
||||
dnl shipped as part of a release package. If it does ship, binaries built here
|
||||
dnl may not be compatible with the minimum supported OS X version.
|
||||
osx_deployment_target_system=
|
||||
|
||||
if test $python_version_major -ne 2; then
|
||||
AC_MSG_ERROR([Assertion failed: building with Python 2.])
|
||||
fi
|
||||
|
||||
dnl sysconfig is only present on Python 2.7 and above.
|
||||
if test $python_version_minor -ge 7; then
|
||||
osx_deployment_target_system=`$PYTHON -c 'import sysconfig; print sysconfig.get_config_var("MACOSX_DEPLOYMENT_TARGET")'`
|
||||
fi
|
||||
|
||||
AC_MSG_RESULT([Populating Python virtualenv])
|
||||
MACOSX_DEPLOYMENT_TARGET=$osx_deployment_target_system \
|
||||
LDFLAGS="${HOST_LDFLAGS}" \
|
||||
CC="${HOST_CC}" CXX="${HOST_CXX}" \
|
||||
CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
|
||||
$PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
|
||||
$_topsrcdir $_topsrcdir/build/virtualenv/packages.txt \
|
||||
|| exit 1
|
||||
|
||||
dnl Load the list of Makefiles to generate.
|
||||
dnl To add new Makefiles, edit allmakefiles.sh.
|
||||
dnl allmakefiles.sh sets the variable, MAKEFILES.
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче