Port mozilla/amo-loadtest to FXA authentication and WebExtensions. (#8089)
* Port mozilla/amo-loadtest to FXA authentication and WebExtensions. This is the first part of many to improve our load-tests and allow them to be run regularly as part of a regression test suite. This commit primarily ports the existing tests over to our repository, updates them to use FxA authentication and uses a WebExtension for the upload test. This only implements the baseline for more work to come in #7744 but it's important to have authentication and the basic infrastructure working correctly. It's also still using the legacy add-on detail and listing pages, that'll obviously change - or be only true for SeaMonkey and Thunderbird related tests since we still have to support them (very low priority though) It also adds a first step of a summary report that links to new-relic. That'll need a bit more tooling and testing but it worked quite well in first tests. Refs #7744 Next steps: * Implement most of the read-only tests from #7744 * Check what needs auth in #7744, implement it properly * Implement database and cache query logging * Implement a unified merged test-report that uses the database and cache query logging and merges it with locust logs (using our new unique request id) * Fix makefile, show dummy-usage for now. * Delete generate-summary script for now * Pin and cleanup dependencies * Reverse all urls
This commit is contained in:
Christopher Grebs
GitHub
Родитель
a31cd814f7
Коммит
e439450c6e
|
|
@ -130,7 +130,7 @@ update: update_deps update_db update_assets
|
|||
reindex:
|
||||
python manage.py reindex $(ARGS)
|
||||
|
||||
ui-tests:
|
||||
setup-ui-tests:
|
||||
rm -rf ./user-media/* ./tmp/*
|
||||
# Reset the database and fake database migrations
|
||||
python manage.py reset_db --noinput
|
||||
|
|
@ -150,10 +150,15 @@ ui-tests:
|
|||
python manage.py waffle_switch activate-autograph-signing on --create
|
||||
python manage.py createsuperuser --email=uitest@mozilla.com --username=uitest --noinput --add-to-supercreate-group --save-api-credentials=tests/ui/variables.json --hostname=olympia-frontend.test
|
||||
|
||||
ui-tests: setup-ui-tests
|
||||
# Generate test add-ons and force a reindex to make sure things are updated
|
||||
python manage.py generate_ui_test_addons
|
||||
python manage.py reindex --force --noinput --wipe
|
||||
|
||||
pip install --progress-bar=off --no-deps -r requirements/uitests.txt
|
||||
|
||||
perf-tests: setup-ui-tests
|
||||
pip install --progress-bar=off --no-deps -r requirements/perftests.txt
|
||||
locust --no-web -c 1 -f tests/performance/locustfile.py --host "http://olympia.test"
|
||||
|
||||
initialize: update_deps initialize_db update_assets populate_data
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
# Required by locust
|
||||
pyzmq==17.0.0 \
|
||||
--hash=sha256:0145ae59139b41f65e047a3a9ed11bbc36e37d5e96c64382fcdff911c4d8c3f0 \
|
||||
--hash=sha256:18de8a02768b1c0b3495ac635b24bd902fafc08befb70a6e68c4d343ccbd6cbd \
|
||||
--hash=sha256:2fb4d745ffe0a65ebf8fd29df093bb5c0ac96a506cb05b9a7b7c94b2524ae7f6 \
|
||||
--hash=sha256:4193cc666591495ab7fe8d24fa8374a35f9775f16dc7c46e03615559e1fc1855 \
|
||||
--hash=sha256:445fed4d71ac48da258ba38f2e29c88c5091124212a4004a0a6a42e6586a7de1 \
|
||||
--hash=sha256:538dfdd9542cf9ff37cd958da03b58d56b53b90800159ea07adc51a8ec7ffcb8 \
|
||||
--hash=sha256:613ac1fc4591b1c6a0a52ce3ed17dbffd6a17e985df504e8b4cdb987f97285b1 \
|
||||
--hash=sha256:630fb21f7474eb9e409a1ad476bf1ec489a69eb021172d422f2485cc3a44cd79 \
|
||||
--hash=sha256:6c3632d2c17cf03ce728ffaa328d45bb053623b3a0aa9747adcde81778d5a4d5 \
|
||||
--hash=sha256:767e1d0b1f7fff1950127abc08c5a5af2754987bc6480c6d641bed6971278a7a \
|
||||
--hash=sha256:863ec1bfa52da6eaa5c4aa59143eeaeb4ef7a076862407a548ec645f25e6d6df \
|
||||
--hash=sha256:a0ecf4c3eccd92f030a4e3e334b9da6fa3ee86be00249343c74e476d70567d0f \
|
||||
--hash=sha256:ad5a8b19b6671b52d30ccfc3a0f4c600e49c4e2dcc88caf4106ed5958dec8d5e \
|
||||
--hash=sha256:b31f2b50ad2920f21b904f5edf66bee324e42bb978df1407ecf381b210d4678e \
|
||||
--hash=sha256:b328c538061757f627d32f7f8885c16f1d2f59f5374e057822f3c8e6cd94c41b \
|
||||
--hash=sha256:b89268020a843d4c3cc04180577ec061fe96d35f267b0b672cb006e4d70560da \
|
||||
--hash=sha256:ba0b43aebf856e5e249250d74c1232d6600b6859328920d12e2ba72a565ab1b1 \
|
||||
--hash=sha256:bdb12b485b3440b5193cd337d27cc126cdfc54ea9f38df237e1ead6216435cbe \
|
||||
--hash=sha256:c30d27c9b35285597b8ef3019f97b9b98457b053f65dcc87a90dfdd4db09ca78 \
|
||||
--hash=sha256:d51eb3902d27d691483243707bfa67972167a70269bbbc172b74eeac4f780a1d \
|
||||
--hash=sha256:e5578ae84bb94e97adadfcb00106a1cb161cb8017f89b01f6c3737f356257811 \
|
||||
--hash=sha256:f35b4cdeffff79357a9d929daa2a8620fb362b2cbeebdc5dd2cf9fcd27c44821 \
|
||||
--hash=sha256:fb983aec4bddee3680a0b7395f99e4595d70d81841370da736c5dc642bad4cd2
|
||||
|
||||
# We need this specific commit until there is a new locust release. Once this happens
|
||||
# This can be pinned to a specific version.
|
||||
-e git+https://github.com/locustio/locust@524ab5203ebc7c4c5c108b641773262ae53fbc40#egg=locust
|
||||
|
|
@ -44,8 +44,9 @@ def application(env, start_response):
|
|||
|
||||
# Initialize Newrelic if we configured it
|
||||
newrelic_ini = getattr(django.conf.settings, 'NEWRELIC_INI', None)
|
||||
newrelic_uses_environment = os.environ.get('NEW_RELIC_LICENSE_KEY', None)
|
||||
|
||||
if newrelic_ini:
|
||||
if newrelic_ini or newrelic_uses_environment:
|
||||
import newrelic.agent
|
||||
try:
|
||||
newrelic.agent.initialize(newrelic_ini)
|
||||
|
|
|
|||
Двоичный файл не отображается.
|
|
@ -0,0 +1,146 @@
|
|||
import logging
|
||||
import random
|
||||
import collections
|
||||
import os
|
||||
import string
|
||||
import re
|
||||
import tempfile
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from shutil import make_archive, rmtree
|
||||
from zipfile import ZipFile
|
||||
|
||||
import lxml
|
||||
|
||||
from fxa.constants import ENVIRONMENT_URLS
|
||||
from fxa.core import Client
|
||||
from fxa.tests.utils import TestEmailAccount
|
||||
|
||||
NAME_REGEX = re.compile('THIS_IS_THE_NAME')
|
||||
|
||||
root_path = os.path.dirname(__file__)
|
||||
data_dir = os.path.join(root_path, 'fixtures')
|
||||
xpis = [os.path.join(data_dir, xpi) for xpi in os.listdir(data_dir)]
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_random():
|
||||
return str(uuid.uuid4())
|
||||
|
||||
|
||||
def submit_url(step):
|
||||
return '/en-US/developers/addon/submit/{step}'.format(step=step)
|
||||
|
||||
|
||||
def get_xpi():
|
||||
return uniqueify_xpi(random.choice(xpis))
|
||||
|
||||
|
||||
@contextmanager
|
||||
def uniqueify_xpi(path):
|
||||
output_dir = tempfile.mkdtemp()
|
||||
try:
|
||||
data_dir = os.path.join(output_dir, 'xpi')
|
||||
output_path = os.path.join(output_dir, 'addon')
|
||||
xpi_name = os.path.basename(path)
|
||||
xpi_path = os.path.join(output_dir, xpi_name)
|
||||
|
||||
with ZipFile(path) as original:
|
||||
original.extractall(data_dir)
|
||||
|
||||
with open(os.path.join(data_dir, 'manifest.json')) as f:
|
||||
manifest_json = f.read()
|
||||
|
||||
manifest_json = NAME_REGEX.sub(get_random(), manifest_json)
|
||||
|
||||
with open(os.path.join(data_dir, 'manifest.json'), 'w') as f:
|
||||
f.write(manifest_json)
|
||||
|
||||
archive_path = make_archive(output_path, 'zip', data_dir)
|
||||
os.rename(archive_path, xpi_path)
|
||||
with open(xpi_path) as f:
|
||||
yield f
|
||||
finally:
|
||||
rmtree(output_dir)
|
||||
|
||||
|
||||
class EventMarker(object):
|
||||
"""
|
||||
Simple event marker that logs on every call.
|
||||
"""
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def _generate_log_message(self):
|
||||
log.info('locust event: {}'.format(self.name))
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
self._generate_log_message()
|
||||
|
||||
|
||||
def install_event_markers():
|
||||
# "import locust" within this scope so that this module is importable by
|
||||
# code running in environments which do not have locust installed.
|
||||
import locust
|
||||
|
||||
# The locust logging format is not necessarily stable, so we use the event
|
||||
# hooks API to implement our own "stable" logging for later programmatic
|
||||
# reference.
|
||||
|
||||
# The events are:
|
||||
|
||||
# * locust_start_hatching
|
||||
# * master_start_hatching
|
||||
# * quitting
|
||||
# * hatch_complete
|
||||
|
||||
# install simple event markers
|
||||
locust.events.locust_start_hatching += EventMarker('locust_start_hatching')
|
||||
locust.events.master_start_hatching += EventMarker('master_start_hatching')
|
||||
locust.events.quitting += EventMarker('quitting')
|
||||
locust.events.hatch_complete += EventMarker('hatch_complete')
|
||||
|
||||
|
||||
def get_fxa_client():
|
||||
fxa_env = os.getenv('FXA_ENV', 'stable')
|
||||
return Client(ENVIRONMENT_URLS[fxa_env]['authentication'])
|
||||
|
||||
|
||||
def get_fxa_account():
|
||||
fxa_client = get_fxa_client()
|
||||
|
||||
account = TestEmailAccount()
|
||||
password = ''.join([random.choice(string.ascii_letters) for i in range(8)])
|
||||
FxAccount = collections.namedtuple('FxAccount', 'email password')
|
||||
fxa_account = FxAccount(email=account.email, password=password)
|
||||
session = fxa_client.create_account(fxa_account.email,
|
||||
fxa_account.password)
|
||||
account.fetch()
|
||||
message = account.wait_for_email(lambda m: 'x-verify-code' in m['headers'])
|
||||
session.verify_email_code(message['headers']['x-verify-code'])
|
||||
return fxa_account, account
|
||||
|
||||
|
||||
def destroy_fxa_account(fxa_account, email_account):
|
||||
email_account.clear()
|
||||
get_fxa_client().destroy_account(fxa_account.email, fxa_account.password)
|
||||
|
||||
|
||||
def get_the_only_form_without_id(response_content):
|
||||
"""
|
||||
Gets the only form on the page that doesn't have an ID.
|
||||
|
||||
A lot of pages (login, registration) have a single form without an ID.
|
||||
This is the one we want. The other forms on the page have IDs so we
|
||||
can ignore them. I'm sure this will break one day.
|
||||
"""
|
||||
html = lxml.html.fromstring(response_content)
|
||||
target_form = None
|
||||
for form in html.forms:
|
||||
if not form.attrib.get('id'):
|
||||
target_form = form
|
||||
if target_form is None:
|
||||
raise ValueError(
|
||||
'Could not find only one form without an ID; found: {}'
|
||||
.format(html.forms))
|
||||
return target_form
|
||||
|
|
@ -0,0 +1,212 @@
|
|||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urlparse
|
||||
import random
|
||||
|
||||
# due to locust sys.path manipulation, we need to re-add the project root.
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
|
||||
|
||||
# Now we can load django settings
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
|
||||
import olympia # noqa
|
||||
from django.conf import settings # noqa
|
||||
from olympia.amo.urlresolvers import reverse
|
||||
|
||||
from locust import HttpLocust, TaskSet, task # noqa
|
||||
import lxml.html # noqa
|
||||
from lxml.html import submit_form # noqa
|
||||
from fxa import oauth as fxa_oauth # noqa
|
||||
|
||||
import helpers # noqa
|
||||
|
||||
logging.Formatter.converter = time.gmtime
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
helpers.install_event_markers()
|
||||
|
||||
MAX_UPLOAD_POLL_ATTEMPTS = 200
|
||||
FXA_CONFIG = settings.FXA_CONFIG[settings.DEFAULT_FXA_CONFIG_NAME]
|
||||
|
||||
|
||||
class UserBehavior(TaskSet):
|
||||
|
||||
def on_start(self):
|
||||
self.fxa_account, self.email_account = helpers.get_fxa_account()
|
||||
|
||||
log.info(
|
||||
'Created {account} for load-tests'
|
||||
.format(account=self.fxa_account))
|
||||
|
||||
def on_stop(self):
|
||||
log.info(
|
||||
'Cleaning up and destroying {account}'
|
||||
.format(account=self.fxa_account))
|
||||
helpers.destroy_fxa_account(self.fxa_account, self.email_account)
|
||||
|
||||
def submit_form(self, form=None, url=None, extra_values=None):
|
||||
if form is None:
|
||||
raise ValueError('form cannot be None; url={}'.format(url))
|
||||
|
||||
def submit(method, form_action_url, values):
|
||||
values = dict(values)
|
||||
if 'csrfmiddlewaretoken' not in values:
|
||||
raise ValueError(
|
||||
'Possibly the wrong form. Could not find '
|
||||
'csrfmiddlewaretoken: {}'.format(repr(values)))
|
||||
|
||||
response = self.client.post(
|
||||
url or form_action_url, values,
|
||||
allow_redirects=False, catch_response=True)
|
||||
|
||||
if response.status_code not in (301, 302):
|
||||
# This probably means the form failed and is displaying
|
||||
# errors.
|
||||
response.failure(
|
||||
'Form submission did not redirect; status={}'
|
||||
.format(response.status_code))
|
||||
|
||||
return submit_form(form, open_http=submit, extra_values=extra_values)
|
||||
|
||||
def login(self, account):
|
||||
log.debug('creating fxa account')
|
||||
fxa_account, email_account = helpers.get_fxa_account()
|
||||
|
||||
log.debug('calling login/start to generate fxa_state')
|
||||
response = self.client.get(
|
||||
reverse('accounts.login_start'),
|
||||
allow_redirects=False)
|
||||
|
||||
params = dict(urlparse.parse_qsl(response.headers['Location']))
|
||||
fxa_state = params['state']
|
||||
|
||||
log.debug('Get browser id session token')
|
||||
fxa_session = helpers.get_fxa_client().login(
|
||||
email=fxa_account.email,
|
||||
password=fxa_account.password)
|
||||
|
||||
oauth_client = fxa_oauth.Client(
|
||||
client_id=FXA_CONFIG['client_id'],
|
||||
client_secret=FXA_CONFIG['client_secret'],
|
||||
server_url=FXA_CONFIG['oauth_host'])
|
||||
|
||||
log.debug('convert browser id session token into oauth code')
|
||||
oauth_code = oauth_client.authorize_code(fxa_session, scope='profile')
|
||||
|
||||
# Now authenticate the user, this will verify the user on the
|
||||
response = self.client.get(
|
||||
reverse('accounts.authenticate'),
|
||||
params={
|
||||
'state': fxa_state,
|
||||
'code': oauth_code,
|
||||
}
|
||||
)
|
||||
|
||||
def logout(self, account):
|
||||
log.debug('Logging out {}'.format(account))
|
||||
self.client.get(reverse('users.logout'))
|
||||
|
||||
def load_upload_form(self):
|
||||
url = helpers.submit_url('upload-unlisted')
|
||||
response = self.client.get(
|
||||
url, allow_redirects=False, catch_response=True)
|
||||
|
||||
if response.status_code == 200:
|
||||
html = lxml.html.fromstring(response.content)
|
||||
return html.get_element_by_id('create-addon')
|
||||
else:
|
||||
more_info = ''
|
||||
if response.status_code in (301, 302):
|
||||
more_info = ('Location: {}'
|
||||
.format(response.headers['Location']))
|
||||
response.failure('Unexpected status: {}; {}'
|
||||
.format(response.status_code, more_info))
|
||||
|
||||
def upload_addon(self, form):
|
||||
url = helpers.submit_url('upload-unlisted')
|
||||
csrfmiddlewaretoken = form.fields['csrfmiddlewaretoken']
|
||||
|
||||
with helpers.get_xpi() as addon_file:
|
||||
response = self.client.post(
|
||||
reverse('devhub.upload'),
|
||||
{'csrfmiddlewaretoken': csrfmiddlewaretoken},
|
||||
files={'upload': addon_file},
|
||||
name='devhub.upload {}'.format(
|
||||
os.path.basename(addon_file.name)),
|
||||
allow_redirects=False,
|
||||
catch_response=True)
|
||||
|
||||
if response.status_code == 302:
|
||||
poll_url = response.headers['location']
|
||||
upload_uuid = self.poll_upload_until_ready(poll_url)
|
||||
if upload_uuid:
|
||||
form.fields['upload'] = upload_uuid
|
||||
self.submit_form(form=form, url=url)
|
||||
else:
|
||||
response.failure('Unexpected status: {}'.format(
|
||||
response.status_code))
|
||||
|
||||
@task(1)
|
||||
def upload(self):
|
||||
self.login(self.fxa_account)
|
||||
|
||||
form = self.load_upload_form()
|
||||
if form:
|
||||
self.upload_addon(form)
|
||||
|
||||
self.logout(self.fxa_account)
|
||||
|
||||
@task(5)
|
||||
def browse(self):
|
||||
self.client.get(reverse('home'))
|
||||
|
||||
response = self.client.get(
|
||||
reverse('browse.extensions'),
|
||||
allow_redirects=False, catch_response=True)
|
||||
|
||||
if response.status_code == 200:
|
||||
html = lxml.html.fromstring(response.content)
|
||||
addon_links = html.cssselect('.item.addon h3 a')
|
||||
url = random.choice(addon_links).get('href')
|
||||
self.client.get(
|
||||
url,
|
||||
name=reverse('addons.detail', kwargs={'addon_id': ':slug'}))
|
||||
else:
|
||||
response.failure('Unexpected status code {}'.format(
|
||||
response.status_code))
|
||||
|
||||
def poll_upload_until_ready(self, url):
|
||||
for i in xrange(MAX_UPLOAD_POLL_ATTEMPTS):
|
||||
response = self.client.get(
|
||||
url, allow_redirects=False,
|
||||
name=reverse('devhub.upload_detail', args=(':uuid',)),
|
||||
catch_response=True)
|
||||
|
||||
try:
|
||||
data = response.json()
|
||||
except ValueError:
|
||||
return response.failure(
|
||||
'Failed to parse JSON when polling. '
|
||||
'Status: {} content: {}'.format(
|
||||
response.status_code, response.content))
|
||||
|
||||
if response.status_code == 200:
|
||||
if data['error']:
|
||||
return response.failure('Unexpected error: {}'.format(
|
||||
data['error']))
|
||||
elif data['validation']:
|
||||
return data['upload']
|
||||
else:
|
||||
return response.failure('Unexpected status: {}'.format(
|
||||
response.status_code))
|
||||
time.sleep(1)
|
||||
else:
|
||||
response.failure('Upload did not complete in {} tries'.format(
|
||||
MAX_UPLOAD_POLL_ATTEMPTS))
|
||||
|
||||
|
||||
class WebsiteUser(HttpLocust):
|
||||
task_set = UserBehavior
|
||||
min_wait = 5000
|
||||
max_wait = 9000
|
||||
Загрузка…
Ссылка в новой задаче