diff --git a/manage.py b/manage.py index 01256aca3..263e7f440 100755 --- a/manage.py +++ b/manage.py @@ -28,6 +28,7 @@ from django.conf import settings # Import for side-effect: configures our logging handlers. from kitsune import log_settings +sys.path.append("/virtualenv/lib/python2.6/site-packages") if __name__ == "__main__": from django.core.management import execute_from_command_line diff --git a/scripts/crontab/gen-crons.py b/scripts/crontab/gen-crons.py index 2967fa899..39353c8d2 100755 --- a/scripts/crontab/gen-crons.py +++ b/scripts/crontab/gen-crons.py @@ -15,7 +15,7 @@ def main(): parser.add_option("-u", "--user", help=("Prefix cron with this user. " "Only define for cron.d style crontabs")) - parser.add_option("-p", "--python", default="/usr/bin/python2.6", + parser.add_option("-p", "--python", default="python", help="Python interpreter to use") (opts, args) = parser.parse_args() @@ -24,15 +24,16 @@ def main(): parser.error("-k must be defined") # To pick up the right PyOpenSSL: - python_path = 'PYTHONPATH=/usr/local/lib64/python2.6/site-packages' + python_path = 'PYTHONPATH+=:/usr/local/lib64/python2.6/site-packages' ctx = { - 'django': 'cd %s; %s %s -W ignore::DeprecationWarning manage.py' % ( + 'django': 'cd %s; source virtualenv/bin/activate; %s %s -W ignore::DeprecationWarning manage.py' % ( opts.kitsune, python_path, opts.python), - 'scripts': 'cd %s; %s %s' % ( + 'scripts': 'cd %s; source virtualenv/bin/activate; %s %s' % ( opts.kitsune, python_path, opts.python), } ctx['cron'] = '%s cron' % ctx['django'] + # Source the venv, don't mess with manage.py ctx['rscripts'] = ctx['scripts'] if opts.user: diff --git a/scripts/travis/install.sh b/scripts/travis/install.sh index c8d7104ea..46b7d29c9 100755 --- a/scripts/travis/install.sh +++ b/scripts/travis/install.sh @@ -8,7 +8,9 @@ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libjpeg.so ~/virtualenv/python2.6/lib/ sudo ln -s /usr/lib/`uname -i`-linux-gnu/libz.so ~/virtualenv/python2.6/lib/ echo "Install Python dependencies" -pip install --allow-external PIL --allow-unverified PIL -r requirements/compiled.txt > /dev/null +python scripts/peep.py install -r requirements/compiled.txt +python scripts/peep.py install -r requirements/requirements_src.txt +python scripts/peep.py install -r requirements/requirements_packages.txt pip install nosenicedots > /dev/null pip freeze echo diff --git a/scripts/update/deploy.py b/scripts/update/deploy.py index 2149d351a..f0e5bbc41 100644 --- a/scripts/update/deploy.py +++ b/scripts/update/deploy.py @@ -15,6 +15,9 @@ from commander.deploy import task, hostgroups import commander_settings as settings +# Setup virtualenv path. +venv_bin_path = os.path.join(settings.SRC_DIR, 'virtualenv', 'bin') +os.environ['PATH'] = venv_bin_path + os.pathsep + os.environ['PATH'] os.environ['DJANGO_SETTINGS_MODULE'] = 'kitsune.settings_local' @@ -25,7 +28,6 @@ def update_code(ctx, tag): ctx.local("git checkout -f %s" % tag) ctx.local("git submodule sync") ctx.local("git submodule update --init --recursive") - ctx.local('find vendor/ -type f -name "*.pyc" | xargs rm -f') @task @@ -40,33 +42,29 @@ def update_locales(ctx): with ctx.lcd(settings.SRC_DIR): ctx.local('date > media/postatus.txt') ctx.local('./scripts/compile-linted-mo.sh | /usr/bin/tee -a media/postatus.txt') - ctx.local('python2.6 manage.py compilejsi18n') + ctx.local('python manage.py compilejsi18n') @task def update_assets(ctx): with ctx.lcd(settings.SRC_DIR): - ctx.local("python2.6 manage.py nunjucks_precompile") - ctx.local("python2.6 manage.py collectstatic --noinput") - ctx.local("LANG=en_US.UTF-8 python2.6 manage.py compress_assets") + ctx.local("python manage.py nunjucks_precompile") + ctx.local("python manage.py collectstatic --noinput") + ctx.local("LANG=en_US.UTF-8 python manage.py compress_assets") @task def db_migrations(ctx): with ctx.lcd(settings.SRC_DIR): - # This is a no-op after the final 232 migration. We should - # remove this at some point once we think all environments and - # contributors have updated. - ctx.local("python2.6 ./vendor/src/schematic/schematic migrations") - - # This performs South migrations. - ctx.local("python2.6 manage.py migrate") + # This runs schematic and south migrations. + ctx.local('schematic migrations') + ctx.local('python manage.py migrate') @task def install_cron(ctx): with ctx.lcd(settings.SRC_DIR): - ctx.local("python2.6 ./scripts/crontab/gen-crons.py -k %s -u apache > /etc/cron.d/.%s" % + ctx.local("python ./scripts/crontab/gen-crons.py -k %s -u apache > /etc/cron.d/.%s" % (settings.WWW_DIR, settings.CRON_NAME)) ctx.local("mv /etc/cron.d/.%s /etc/cron.d/%s" % (settings.CRON_NAME, settings.CRON_NAME)) @@ -107,9 +105,8 @@ def update_info(ctx): ctx.local("git log -3") ctx.local("git status") ctx.local("git submodule status") - # TODO: Nix this when we nix schematic. - ctx.local("python2.6 ./vendor/src/schematic/schematic -v migrations/") - ctx.local("python2.6 manage.py migrate --list") + print '\n\n\n\n\n' + os.environ['PATH'] + '\n\n\n\n\n' + ctx.local("python manage.py migrate --list") with ctx.lcd("locale"): ctx.local("svn info") ctx.local("svn status") @@ -117,9 +114,34 @@ def update_info(ctx): ctx.local("git rev-parse HEAD > media/revision.txt") +@task +def setup_dependencies(ctx): + with ctx.lcd(settings.SRC_DIR): + # Creating a virtualenv tries to open virtualenv/bin/python for + # writing, but because virtualenv is using it, it fails. + # So we delete it and let virtualenv create a new one. + ctx.local('rm -f virtualenv/bin/python') + ctx.local('virtualenv --no-site-packages virtualenv') + + # Activate virtualenv to append to path. + activate_env = os.path.join(settings.SRC_DIR, 'virtualenv', 'bin', 'activate_this.py') + execfile(activate_env, dict(__file__=activate_env)) + + ctx.local('python scripts/peep.py install -r requirements/compiled.txt') + ctx.local('python scripts/peep.py install -r requirements/git.txt') + ctx.local('python scripts/peep.py install -r requirements/pypi.txt') + ctx.local('virtualenv --relocatable virtualenv') + + # Fix lib64 symlink to be relative instead of absolute. + ctx.local('rm -f virtualenv/lib64') + with ctx.lcd('virtualenv'): + ctx.local('ln -s lib lib64') + + @task def pre_update(ctx, ref=settings.UPDATE_REF): update_code(ref) + setup_dependencies() update_info() diff --git a/scripts/update/migrate.sh b/scripts/update/migrate.sh new file mode 100755 index 000000000..b6d40b57a --- /dev/null +++ b/scripts/update/migrate.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Run migrations for kitsune + +source virtualenv/bin/activate +python schematic migrations +python manage.py migrate diff --git a/scripts/update/setup_dependencies.sh b/scripts/update/setup_dependencies.sh new file mode 100755 index 000000000..126e6cc0c --- /dev/null +++ b/scripts/update/setup_dependencies.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# This creates a virtualenv and installs our requirements into it. +# CWD should be the project folder, i.e. kitsune/. + +set -e + +virtualenv --system-site-packages virtualenv + +source virtualenv/bin/activate +python scripts/peep.py install -r requirements/requirements_src.txt +python scripts/peep.py install -r requirements/requirements_packages.txt + +# Fix path issues for when we rsync.. +virtualenv --relocatable virtualenv diff --git a/wsgi/kitsune.wsgi b/wsgi/kitsune.wsgi index 2aea3c914..d97e84474 100644 --- a/wsgi/kitsune.wsgi +++ b/wsgi/kitsune.wsgi @@ -19,14 +19,18 @@ if newrelic: # Remember when mod_wsgi loaded this file so we can track it in nagios. wsgi_loaded = datetime.now() -# Add the zamboni dir to the python path so we can import manage. +# Add kitsune to the python path wsgidir = os.path.dirname(__file__) site.addsitedir(os.path.abspath(os.path.join(wsgidir, '../'))) # For django-celery os.environ['CELERY_LOADER'] = 'django' -# manage /vendor to the Python path. +# Activate virtualenv +activate_env = os.path.abspath(os.path.join(wsgidir, "../virtualenv/bin/activate_this.py")) +execfile(activate_env, dict(__file__=activate_env)) + +# Import for side-effects: set-up import manage import django.conf