Bug 774106 - Don't use Makefile.in to populate virtualenv; r=glandium

allmakefiles.sh

@@ -30,7 +30,6 @@ build/Makefile
 build/pgo/Makefile
 build/pgo/blueprint/Makefile
 build/pgo/js-input/Makefile
-build/virtualenv/Makefile
 config/Makefile
 config/autoconf.mk
 config/nspr/Makefile

build/virtualenv/Makefile.in

@@ -1,35 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-DEPTH     = ../..
-topsrcdir = @top_srcdir@
-srcdir    = @srcdir@
-VPATH     = @srcdir@
-
-include $(DEPTH)/config/autoconf.mk
-
-# Paths here are topsrcdir-relative, and
-# must be in dependency-order.
-setuptools_packages := \
-  other-licenses/simplejson-2.1.1 \
-  testing/mozbase/manifestdestiny \
-  testing/mozbase/mozinfo \
-  testing/mozbase/mozinstall \
-  testing/mozbase/mozlog \
-  testing/mozbase/mozprocess \
-  testing/mozbase/mozprofile \
-  testing/mozbase/mozrunner \
-  build/pylib/blessings \
-  $(NULL)
-
-
-define install_setuptools_package
-cd $(topsrcdir)/$(1)/; CFLAGS="$(HOST_CFLAGS)" LDFLAGS="$(HOST_LDFLAGS)" CXXFLAGS="$(HOST_CXXFLAGS)" $(PYTHON) setup.py develop
-
-endef
-
-default::
-	$(foreach package,$(setuptools_packages),$(call install_setuptools_package,$(package)))
-
-include $(topsrcdir)/config/rules.mk

build/virtualenv/packages.txt

@@ -0,0 +1,9 @@
+setup.py:other-licenses/simplejson-2.1.1:develop
+setup.py:testing/mozbase/manifestdestiny:develop
+setup.py:testing/mozbase/mozinfo:develop
+setup.py:testing/mozbase/mozinstall:develop
+setup.py:testing/mozbase/mozlog:develop
+setup.py:testing/mozbase/mozprocess:develop
+setup.py:testing/mozbase/mozprofile:develop
+setup.py:testing/mozbase/mozrunner:develop
+setup.py:build/pylib/blessings:develop

build/virtualenv/populate_virtualenv.py

@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains code for populating the virtualenv environment for
+# Mozilla's build system. It is typically called as part of configure.
+
+import os.path
+import subprocess
+import sys
+
+def populate_virtualenv(top_source_directory, manifest_filename):
+    """Populate the virtualenv from the contents of a manifest.
+
+    The manifest file consists of colon-delimited fields. The first field
+    specifies the action. The remaining fields are arguments to that action.
+    The following actions are supported:
+
+      setup.py -- Invoke setup.py for a package. Expects the arguments:
+        1. relative path directory containing setup.py.
+        2. argument(s) to setup.py. e.g. "develop". Each program argument is
+           delimited by a colon. Arguments with colons are not yet supported.
+
+    Note that the Python interpreter running this function should be the one
+    from the virtualenv. If it is the system Python or if the environment is
+    not configured properly, packages could be installed into the wrong place.
+    This is how virtualenv's work.
+    """
+    packages = []
+    fh = open(manifest_filename, 'rU')
+    for line in fh:
+        packages.append(line.rstrip().split(':'))
+    fh.close()
+
+    for package in packages:
+        if package[0] == 'setup.py':
+            assert len(package) >= 2
+
+            call_setup(os.path.join(top_source_directory, package[1]),
+                package[2:])
+
+def call_setup(directory, arguments):
+    """Calls setup.py in a directory."""
+    setup = os.path.join(directory, 'setup.py')
+
+    program = [sys.executable, setup]
+    program.extend(arguments)
+
+    # We probably could call the contents of this file inside the context of
+    # this interpreter using execfile() or similar. However, if global
+    # variables like sys.path are adjusted, this could cause all kinds of
+    # havoc. While this may work, invoking a new process is safer.
+    result = subprocess.call(program, cwd=directory)
+
+    if result != 0:
+        raise Exception('Error installing package: %s' % directory)
+
+# configure invokes us with /path/to/topsrcdir and /path/to/manifest
+if __name__ == '__main__':
+    assert len(sys.argv) == 3
+
+    populate_virtualenv(sys.argv[1], sys.argv[2])
+    sys.exit(0)

client.mk

@@ -254,10 +254,10 @@ CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status)
 CONFIG_CACHE  = $(wildcard $(OBJDIR)/config.cache)
 
 EXTRA_CONFIG_DEPS := \
-	$(TOPSRCDIR)/aclocal.m4 \
-	$(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
-	$(TOPSRCDIR)/js/src/aclocal.m4 \
-	$(NULL)
+  $(TOPSRCDIR)/aclocal.m4 \
+  $(wildcard $(TOPSRCDIR)/build/autoconf/*.m4) \
+  $(TOPSRCDIR)/js/src/aclocal.m4 \
+  $(NULL)
 
 $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	@$(PYTHON) $(TOPSRCDIR)/js/src/config/check-sync-dirs.py $(TOPSRCDIR)/js/src/build $(TOPSRCDIR)/build
@@ -265,16 +265,16 @@ $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	cd $(@D); $(AUTOCONF)
 
 CONFIG_STATUS_DEPS := \
-	$(wildcard \
-        $(CONFIGURES) \
-        $(TOPSRCDIR)/allmakefiles.sh \
-        $(TOPSRCDIR)/nsprpub/configure \
-        $(TOPSRCDIR)/config/milestone.txt \
-        $(TOPSRCDIR)/js/src/config/milestone.txt \
-        $(TOPSRCDIR)/browser/config/version.txt \
-        $(TOPSRCDIR)/*/confvars.sh \
-	) \
-	$(NULL)
+  $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
+  $(CONFIGURES) \
+  $(TOPSRCDIR)/allmakefiles.sh \
+  $(TOPSRCDIR)/nsprpub/configure \
+  $(TOPSRCDIR)/config/milestone.txt \
+  $(TOPSRCDIR)/js/src/config/milestone.txt \
+  $(TOPSRCDIR)/browser/config/version.txt \
+  $(TOPSRCDIR)/build/virtualenv/packages.txt \
+  $(TOPSRCDIR)/build/virtualenv/populate_virtualenv.py \
+  $(NULL)
 
 CONFIGURE_ENV_ARGS += \
   MAKE="$(MAKE)" \

configure.in

@@ -8780,6 +8780,15 @@ esac
 
 AC_SUBST(PYTHON)
 
+# Populate the virtualenv
+AC_MSG_RESULT([Populating Python virtualenv])
+MACOSX_DEPLOYMENT_TARGET= LDFLAGS="${HOST_LDFLAGS}" \
+  CC="${CC}" CXX="${CXX}" \
+  CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" \
+  $PYTHON $_topsrcdir/build/virtualenv/populate_virtualenv.py \
+    $_topsrcdir $_topsrcdir/build/virtualenv/packages.txt \
+  || exit 1
+
 dnl Load the list of Makefiles to generate.
 dnl   To add new Makefiles, edit allmakefiles.sh.
 dnl   allmakefiles.sh sets the variable, MAKEFILES.
@@ -8853,10 +8862,6 @@ if test -n "$MOZ_WEBRTC"; then
    fi
 fi
 
-# Populate the virtualenv
-AC_MSG_RESULT([Populating Python virtualenv])
-$MAKE -C build/virtualenv MACOSX_DEPLOYMENT_TARGET=  || exit 1
-
 # Generate a JSON config file for unittest harnesses etc to read
 # build configuration details from in a standardized way.
 OS_TARGET=${OS_TARGET} TARGET_CPU=${TARGET_CPU} MOZ_DEBUG=${MOZ_DEBUG} \