[bug 1035319] Updated travis and deploy scripts.

manage.py

@@ -28,6 +28,7 @@ from django.conf import settings
 # Import for side-effect: configures our logging handlers.
 from kitsune import log_settings
 
+sys.path.append("/virtualenv/lib/python2.6/site-packages")
 
 if __name__ == "__main__":
     from django.core.management import execute_from_command_line

scripts/crontab/gen-crons.py

@@ -15,7 +15,7 @@ def main():
     parser.add_option("-u", "--user",
                       help=("Prefix cron with this user. "
                            "Only define for cron.d style crontabs"))
-    parser.add_option("-p", "--python", default="/usr/bin/python2.6",
+    parser.add_option("-p", "--python", default="python",
                       help="Python interpreter to use")
 
     (opts, args) = parser.parse_args()
@@ -24,15 +24,16 @@ def main():
         parser.error("-k must be defined")
 
     # To pick up the right PyOpenSSL:
-    python_path = 'PYTHONPATH=/usr/local/lib64/python2.6/site-packages'
+    python_path = 'PYTHONPATH+=:/usr/local/lib64/python2.6/site-packages'
 
     ctx = {
-        'django': 'cd %s; %s %s -W ignore::DeprecationWarning manage.py' % (
+        'django': 'cd %s; source virtualenv/bin/activate; %s %s -W ignore::DeprecationWarning manage.py' % (
             opts.kitsune, python_path, opts.python),
-        'scripts': 'cd %s; %s %s' % (
+        'scripts': 'cd %s; source virtualenv/bin/activate; %s %s' % (
             opts.kitsune, python_path, opts.python),
     }
     ctx['cron'] = '%s cron' % ctx['django']
+    # Source the venv, don't mess with manage.py
     ctx['rscripts'] = ctx['scripts']
 
     if opts.user:

scripts/travis/install.sh

@@ -8,7 +8,9 @@ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.6/lib/
 sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.6/lib/
 
 echo "Install Python dependencies"
-pip install --allow-external PIL --allow-unverified PIL -r requirements/compiled.txt > /dev/null
+python scripts/peep.py install -r requirements/compiled.txt
+python scripts/peep.py install -r requirements/requirements_src.txt
+python scripts/peep.py install -r requirements/requirements_packages.txt
 pip install nosenicedots > /dev/null
 pip freeze
 echo

scripts/update/deploy.py

@@ -15,6 +15,9 @@ from commander.deploy import task, hostgroups
 import commander_settings as settings
 
 
+# Setup virtualenv path.
+venv_bin_path = os.path.join(settings.SRC_DIR, 'virtualenv', 'bin')
+os.environ['PATH'] = venv_bin_path + os.pathsep + os.environ['PATH']
 os.environ['DJANGO_SETTINGS_MODULE'] = 'kitsune.settings_local'
 
 
@@ -25,7 +28,6 @@ def update_code(ctx, tag):
         ctx.local("git checkout -f %s" % tag)
         ctx.local("git submodule sync")
         ctx.local("git submodule update --init --recursive")
-        ctx.local('find vendor/ -type f -name "*.pyc" | xargs rm -f')
 
 
 @task
@@ -40,33 +42,29 @@ def update_locales(ctx):
     with ctx.lcd(settings.SRC_DIR):
         ctx.local('date > media/postatus.txt')
         ctx.local('./scripts/compile-linted-mo.sh | /usr/bin/tee -a media/postatus.txt')
-        ctx.local('python2.6 manage.py compilejsi18n')
+        ctx.local('python manage.py compilejsi18n')
 
 
 @task
 def update_assets(ctx):
     with ctx.lcd(settings.SRC_DIR):
-        ctx.local("python2.6 manage.py nunjucks_precompile")
-        ctx.local("python2.6 manage.py collectstatic --noinput")
-        ctx.local("LANG=en_US.UTF-8 python2.6 manage.py compress_assets")
+        ctx.local("python manage.py nunjucks_precompile")
+        ctx.local("python manage.py collectstatic --noinput")
+        ctx.local("LANG=en_US.UTF-8 python manage.py compress_assets")
 
 
 @task
 def db_migrations(ctx):
     with ctx.lcd(settings.SRC_DIR):
-        # This is a no-op after the final 232 migration. We should
-        # remove this at some point once we think all environments and
-        # contributors have updated.
-        ctx.local("python2.6 ./vendor/src/schematic/schematic migrations")
-
-        # This performs South migrations.
-        ctx.local("python2.6 manage.py migrate")
+        # This runs schematic and south migrations.
+        ctx.local('schematic migrations')
+        ctx.local('python manage.py migrate')
 
 
 @task
 def install_cron(ctx):
     with ctx.lcd(settings.SRC_DIR):
-        ctx.local("python2.6 ./scripts/crontab/gen-crons.py -k %s -u apache > /etc/cron.d/.%s" %
+        ctx.local("python ./scripts/crontab/gen-crons.py -k %s -u apache > /etc/cron.d/.%s" %
                   (settings.WWW_DIR, settings.CRON_NAME))
         ctx.local("mv /etc/cron.d/.%s /etc/cron.d/%s" % (settings.CRON_NAME, settings.CRON_NAME))
 
@@ -107,9 +105,8 @@ def update_info(ctx):
         ctx.local("git log -3")
         ctx.local("git status")
         ctx.local("git submodule status")
-        # TODO: Nix this when we nix schematic.
-        ctx.local("python2.6 ./vendor/src/schematic/schematic -v migrations/")
-        ctx.local("python2.6 manage.py migrate --list")
+        print '\n\n\n\n\n' + os.environ['PATH'] + '\n\n\n\n\n'
+        ctx.local("python manage.py migrate --list")
         with ctx.lcd("locale"):
             ctx.local("svn info")
             ctx.local("svn status")
@@ -117,9 +114,34 @@ def update_info(ctx):
         ctx.local("git rev-parse HEAD > media/revision.txt")
 
 
+@task
+def setup_dependencies(ctx):
+    with ctx.lcd(settings.SRC_DIR):
+        # Creating a virtualenv tries to open virtualenv/bin/python for
+        # writing, but because virtualenv is using it, it fails.
+        # So we delete it and let virtualenv create a new one.
+        ctx.local('rm -f virtualenv/bin/python')
+        ctx.local('virtualenv --no-site-packages virtualenv')
+
+        # Activate virtualenv to append to path.
+        activate_env = os.path.join(settings.SRC_DIR, 'virtualenv', 'bin', 'activate_this.py')
+        execfile(activate_env, dict(__file__=activate_env))
+
+        ctx.local('python scripts/peep.py install -r requirements/compiled.txt')
+        ctx.local('python scripts/peep.py install -r requirements/git.txt')
+        ctx.local('python scripts/peep.py install -r requirements/pypi.txt')
+        ctx.local('virtualenv --relocatable virtualenv')
+
+        # Fix lib64 symlink to be relative instead of absolute.
+        ctx.local('rm -f virtualenv/lib64')
+        with ctx.lcd('virtualenv'):
+            ctx.local('ln -s lib lib64')
+
+
 @task
 def pre_update(ctx, ref=settings.UPDATE_REF):
     update_code(ref)
+    setup_dependencies()
     update_info()
 
 

scripts/update/migrate.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Run migrations for kitsune
+
+source virtualenv/bin/activate
+python schematic migrations
+python manage.py migrate

scripts/update/setup_dependencies.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# This creates a virtualenv and installs our requirements into it.
+# CWD should be the project folder, i.e. kitsune/.
+
+set -e
+
+virtualenv --system-site-packages virtualenv
+
+source virtualenv/bin/activate
+python scripts/peep.py install -r requirements/requirements_src.txt
+python scripts/peep.py install -r requirements/requirements_packages.txt
+
+# Fix path issues for when we rsync..
+virtualenv --relocatable virtualenv

wsgi/kitsune.wsgi

@@ -19,14 +19,18 @@ if newrelic:
 # Remember when mod_wsgi loaded this file so we can track it in nagios.
 wsgi_loaded = datetime.now()
 
-# Add the zamboni dir to the python path so we can import manage.
+# Add kitsune to the python path
 wsgidir = os.path.dirname(__file__)
 site.addsitedir(os.path.abspath(os.path.join(wsgidir, '../')))
 
 # For django-celery
 os.environ['CELERY_LOADER'] = 'django'
 
-# manage /vendor to the Python path.
+# Activate virtualenv
+activate_env = os.path.abspath(os.path.join(wsgidir, "../virtualenv/bin/activate_this.py"))
+execfile(activate_env, dict(__file__=activate_env))
+
+# Import for side-effects: set-up
 import manage
 
 import django.conf