From 7cce7b2fa7eb067586c8ae369008e8c3a94edc43 Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Thu, 20 Oct 2016 20:04:47 +0100 Subject: [PATCH] Bug 1308354 - Remove SCL3-specific code now we're on Heroku --- .../update/commander_settings.py-example | 16 -- deployment/update/update.py | 146 ------------------ setup.cfg | 2 +- treeherder/config/settings.py | 22 +-- 4 files changed, 8 insertions(+), 178 deletions(-) delete mode 100644 deployment/update/commander_settings.py-example delete mode 100644 deployment/update/update.py diff --git a/deployment/update/commander_settings.py-example b/deployment/update/commander_settings.py-example deleted file mode 100644 index 0f2e15323..000000000 --- a/deployment/update/commander_settings.py-example +++ /dev/null @@ -1,16 +0,0 @@ -# Example Commander configuration file for Chief deployment. -# Copy to commander_settings.py & adjust as required. -SRC_DIR = '/data/treeherder/src/treeherder.mozilla.org/' -BIN_DIR = '/usr/bin' -SBIN_DIR = '/sbin' - -DEPLOY_SCRIPT = '/data/treeherder/deploy -n treeherder.mozilla.org' -REMOTE_UPDATE_SCRIPT = '/data/bin/update-www.sh' - -WEB_HOSTGROUP = 'treeherder-web' -ETL_HOSTGROUP = 'treeherder-etl' -LOG_HOSTGROUP = 'treeherder-processors' -RABBIT_HOSTGROUP = 'treeherder-rabbit' - -UPDATE_REF = 'master' -SSH_KEY = '/root/.ssh/id_rsa_treeherder_updater' diff --git a/deployment/update/update.py b/deployment/update/update.py deleted file mode 100644 index 4fbda8202..000000000 --- a/deployment/update/update.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -Deploy this project in stage/production. - -Requires commander_ which is installed on the systems that need it. - -.. _commander: https://github.com/oremj/commander -""" - -import os -import requests -import sys - -from commander.deploy import hostgroups, task - -sys.path.append(os.path.dirname(os.path.abspath(__file__))) -import commander_settings as settings # noqa - -env_file = os.path.join(settings.SRC_DIR, 'treeherder-env.sh') -th_service_src = os.path.join(settings.SRC_DIR, 'treeherder-service') - -is_prod = 'treeherder.mozilla.org' in settings.SRC_DIR - - -def run_local_with_env(ctx, cmd): - # For commands run from the admin node, we have to manually set the environment - # variables, since the admin node is shared by both stage and prod. - ctx.local("source {} && {}".format(env_file, cmd)) - - -@task -def pre_update(ctx, ref=settings.UPDATE_REF): - # Update the code to a specific git reference (branch/tag/sha) and write - # info about the current repository state to a publicly visible file. - with ctx.lcd(th_service_src): - ctx.local('git fetch --quiet origin %s' % ref) - ctx.local('git reset --hard FETCH_HEAD') - ctx.local('find . -type f -name "*.pyc" -delete') - ctx.local('git status -s') - - -@task -def update(ctx): - # Create/populate a virtualenv that will be rsynced later along with the source. - with ctx.lcd(settings.SRC_DIR): - activate_script = os.path.join(settings.SRC_DIR, 'venv', 'bin', 'activate_this.py') - # We reuse the virtualenv to speed up deploys. - if not os.path.exists(activate_script): - ctx.local('virtualenv --python=python2.7 venv') - # Activate virtualenv. - execfile(activate_script, dict(__file__=activate_script)) - # Install requirements using pip v8's new hash-checking mode. - with ctx.lcd(th_service_src): - ctx.local('pip2.7 install --require-hashes -r requirements/common.txt') - # Make the virtualenv relocatable since paths are hard-coded by default. - ctx.local('virtualenv --relocatable venv') - # Fix lib64 symlink to be relative instead of absolute. - with ctx.lcd('venv'): - ctx.local('rm -f lib64') - ctx.local('ln -s lib lib64') - - with ctx.lcd(th_service_src): - # Install nodejs non-dev packages, needed for the grunt build. - ctx.local("npm install --production") - # Generate the UI assets in the `dist/` directory. - ctx.local("./node_modules/.bin/grunt build --production") - # Make the current Git revision accessible at /revision.txt - ctx.local("git rev-parse HEAD > dist/revision.txt") - # Generate gzipped versions of files that would benefit from compression, that - # WhiteNoise can then serve in preference to the originals. This is required - # since WhiteNoise's Django storage backend only gzips assets handled by - # collectstatic, and so does not affect files in the `dist/` directory. - ctx.local("python2.7 -m whitenoise.compress dist") - # Run Django system checks. - # Replace awk with `--fail-level WARNING` once we're using Django 1.10, since in - # previous versions an exit code of 1 is hard-coded to only ERROR and above: - # https://github.com/django/django/commit/287532588941d2941e19c4cd069bcbd8af889203 - run_local_with_env(ctx, 'python2.7 manage.py check --deploy 2>&1 | awk "/^WARNINGS/{err=1} {print} END{exit err}"') - # Collect the static files (eg for the Persona or Django admin UI) - run_local_with_env(ctx, "python2.7 manage.py collectstatic --noinput") - # Update the database schema, if necessary. - run_local_with_env(ctx, "python2.7 manage.py migrate --noinput") - # Update reference data & tasks config from the in-repo fixtures. - run_local_with_env(ctx, "python2.7 manage.py load_initial_data") - # Populate the datasource table and create the connected databases. - run_local_with_env(ctx, "python2.7 manage.py init_datasources") - - -@task -def deploy(ctx): - # Use the local, IT-written deploy script to check in changes. - ctx.local(settings.DEPLOY_SCRIPT) - # Rsync the updated code to the nodes & restart processes. These are - # separated out into their own functions, since the IRC bot output includes - # the task function name which is useful given how long these steps take. - deploy_rabbit() - deploy_web_app() - deploy_etl() - deploy_log() - ping_newrelic() - - -@task -def deploy_rabbit(ctx): - deploy_nodes(ctx, settings.RABBIT_HOSTGROUP, 'rabbit') - - -@task -def deploy_web_app(ctx): - deploy_nodes(ctx, settings.WEB_HOSTGROUP, 'web') - - -@task -def deploy_etl(ctx): - deploy_nodes(ctx, settings.ETL_HOSTGROUP, 'etl') - - -@task -def deploy_log(ctx): - deploy_nodes(ctx, settings.LOG_HOSTGROUP, 'log') - - -def deploy_nodes(ctx, hostgroup, node_type): - # Run the remote update script on each node in the specified hostgroup. - @hostgroups(hostgroup, remote_kwargs={'ssh_key': settings.SSH_KEY}) - def rsync_code(ctx): - ctx.remote(settings.REMOTE_UPDATE_SCRIPT) - - rsync_code() - env_flag = '-p' if is_prod else '-s' - ctx.local('/root/bin/restart-jobs %s %s' % (env_flag, node_type)) - - -@task -def ping_newrelic(ctx): - data = { - 'deployment[application_id]': settings.NEW_RELIC_APP_ID, - 'deployment[user]': 'Chief', - } - headers = {'x-api-key': settings.NEW_RELIC_API_KEY} - r = requests.post('https://api.newrelic.com/deployments.xml', - data=data, headers=headers, timeout=30) - try: - r.raise_for_status() - except requests.exceptions.HTTPError: - print("HTTPError {} notifying New Relic: {}".format(r.status_code, r.text)) - raise diff --git a/setup.cfg b/setup.cfg index 7f1abd305..a5a5b815a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,7 @@ ignore = E121,E123,E125,E126,E129,E226,E24,E704,E501,F403 max-line-length = 140 [isort] -skip = .git,__pycache__,.vagrant,node_modules,migrations,update.py,wsgi.py +skip = .git,__pycache__,.vagrant,node_modules,migrations,wsgi.py multi_line_output = 1 force_grid_wrap = true line_length = 140 diff --git a/treeherder/config/settings.py b/treeherder/config/settings.py index 9a24193eb..328723eb7 100644 --- a/treeherder/config/settings.py +++ b/treeherder/config/settings.py @@ -13,12 +13,10 @@ env = environ.Env() def server_supports_tls(url): hostname = urlparse(url).hostname - # Services such as RabbitMQ/Elasticsearch running on Travis/Vagrant - # or in SCl3 do not support TLS. We could try adding locally using - # self-signed certs, but until Travis has support it's not overly useful. - if hostname == 'localhost' or hostname.endswith('.scl3.mozilla.com'): - return False - return True + # Services such as RabbitMQ/Elasticsearch running on Travis do not yet have TLS + # certificates set up. We could try using TLS locally using self-signed certs, + # but until Travis has support it's not overly useful. + return hostname != 'localhost' TREEHERDER_MEMCACHED = env("TREEHERDER_MEMCACHED", default="127.0.0.1:11211") TREEHERDER_MEMCACHED_KEY_PREFIX = env("TREEHERDER_MEMCACHED_KEY_PREFIX", default="treeherder") @@ -411,12 +409,6 @@ SILENCED_SYSTEM_CHECKS = [ 'security.W017', ] -# Disable the libmysqlclient TLS system check on SCl3, since we're moving to -# Heroku soon, so it's not worth the hassle of trying to get it updated from -# the ancient 5.1.X, given the DB is behind a VPN and so doesn't use TLS anyway. -if '.scl3.mozilla.com' in env('DATABASE_URL'): - SILENCED_SYSTEM_CHECKS.append('treeherder.E001') - # Enable integration between autoclassifier and jobs AUTOCLASSIFY_JOBS = env.bool("AUTOCLASSIFY_JOBS", default=True) # Ordered list of matcher classes to use during autoclassification @@ -581,15 +573,15 @@ KEY_PREFIX = TREEHERDER_MEMCACHED_KEY_PREFIX # Celery broker setup BROKER_URL = env('BROKER_URL') -# Force Celery to use TLS when appropriate (ie if not localhost or SCL3), +# Force Celery to use TLS when appropriate (ie if not localhost), # rather than relying on `BROKER_URL` having `amqps://` or `?ssl=` set. # This is required since CloudAMQP's automatically defined URL uses neither. if server_supports_tls(BROKER_URL): BROKER_USE_SSL = True # This code handles the memcachier service on heroku. -# TODO: Once no longer on SCL3, stop special-casing Heroku and use newer -# best practices from https://www.memcachier.com/documentation#django. +# TODO: Stop special-casing Heroku and use newer best practices from: +# https://www.memcachier.com/documentation#django. if env.bool('IS_HEROKU', default=False): # Prefs taken from: # https://github.com/rdegges/django-heroku-memcacheify/blob/v1.0.0/memcacheify.py#L30-L39