2015-08-26 20:33:42 +03:00
|
|
|
from django import http, test
|
|
|
|
from django.core.cache import cache
|
|
|
|
from django.utils import translation
|
|
|
|
|
|
|
|
import caching
|
2014-11-19 19:36:40 +03:00
|
|
|
import pytest
|
Default to 'and' operator for match queries. Remove slug search, prioritize exact matches more. (#7303)
* Default to 'and' operator for match queries. Remove slug search, prioritize exact matches more.
References many "component: search" issues. What I tested with a
database of all public add-ons:
Example searches:
tab center redux - should find "Tab Center Redux" while "Tab Mix Plus" is probably second and "Redux DevTools" 4th or so
Open Image in New Tab -> should find "Open Image in New Tab" while "Open Bookmarks in New Tab" should be 2nd or 3rd
CoinHive -> Finds "Coinhive Blocker", "CoinBlock" (prefix search) and "NoMiners" (description)
Privacy -> Finds "Privacy Badger", "Privacy Pass", "Privacy Settings", "Google Privacy" (probably 4th or so) and "Blur" (summary + description + many users). Scores "Ghostery" on the first page but ranks it in the middle
firebu -> Finds "Firebug", "Firebug Autocompleter", "Firefinder for Firebug"
fireb -> Scores "Fire Drag" first, puts "Firebug" approximately 3rd or so
Menu Wizzard -> Finds "Menu Wizard" (fuzzy, typo) first, then "Add-ons Manager Context Menu" apparently because it matches good in the title and has many users
Frame Demolition -> Finds "Frame Demolition"
Demolition -> Finds only "Frame Demolition", same for "Demolation" (typo)
reStyle -> Finds "reStyle" and scores a few add-ons that match on "restore" next since the term is similar
MegaUpload DownloadHelper -> finds "MegaUpload DownloadHelper" first, scores "Video DownloadHelper" and "RadpidShare DownloadHelper" next. Doesn't find "Popup Blocker" anymore as currently happening on -prod
MegaUpload -> only finds "MegaUpload DownloadHelper" and nothing else
No Flash -> Scores "No Flash" first, then depending on users "Download Flash and Video", "YouTube Flash Video Player" and "YouTube Flash Player" (not necessarily in that order)
Disable Hello, Pocket & Reader+ -> finds "Disable Hello, Pocket & Reader+" first (yeay!), then scores "Reader", "Disable WebRTC" and "In My Pocket" next similarly to what's happening on -prod currently
Not working yet:
privacybadger -> "Privacy Badger" -> will probably need some kind of ngram filtering and analyzing (#591)
eyes -> 'decentraleyes' -> Not sure this should actually work, will probably need some more analyzing too (#591)
Not sure if it's specifically only because of these changes but #3248 is fixed.
This potentially fixes #7244, #6891, #6837, #6417, mozilla/addons#359.
Not sure if this fixes #mozilla/addons#567 but the results look much more promising and the amount of results doesn't explode here. I only have 2.8k add-ons for testing though so I'm not too sure.
And might be relevant to #6137.
This is a big step towards #2661, I doubt we can call this fixed though.
* Fix name tests
* Speed up ES tests, make scoring results more predictable by using only one shard and one replica
* Add tests
* Only test on new apiv3 based search.
* Fixup 'get_results'
* Remove debug print
* Isort imports
* Fix test settings, I'm blind.
* Adapt number of shards to what we define in settings_test
* Test search for grapple
* Remove property filtering in legacy api search, fix tests, fix flake8
* Add comment explaining shard config
* Fix tests again, add default platform, fix total counts again.
* Try to do some fixture cleanup, let's see...
* More test refactoring, make use of dfs-query-then-fetch during tests, allow us to test this later too via a waffle flag.
* Don't use waffle flag for legacy search, only for apiv3 searches.
* Fix unused import, fix usage of 'params'
* Use query-then-fetch in a regular addons-view tool, fix a few more tests to be able to handle the waffle-flag
* Create the dfs query then fetch flag only in ESTestCase, delete it properly. More serializer fixes
* precache the waffle flag for autocomplete tests too
* Minor cleanups
* Add docs, fix code style
* Add a todo
* Fix codestyle
2018-01-30 08:26:30 +03:00
|
|
|
|
2015-08-31 10:22:37 +03:00
|
|
|
from multidb import pinning
|
2014-11-19 19:36:40 +03:00
|
|
|
|
2017-03-06 19:19:34 +03:00
|
|
|
from olympia import amo, core
|
2015-12-03 00:52:47 +03:00
|
|
|
from olympia.access.models import Group, GroupUser
|
Default to 'and' operator for match queries. Remove slug search, prioritize exact matches more. (#7303)
* Default to 'and' operator for match queries. Remove slug search, prioritize exact matches more.
References many "component: search" issues. What I tested with a
database of all public add-ons:
Example searches:
tab center redux - should find "Tab Center Redux" while "Tab Mix Plus" is probably second and "Redux DevTools" 4th or so
Open Image in New Tab -> should find "Open Image in New Tab" while "Open Bookmarks in New Tab" should be 2nd or 3rd
CoinHive -> Finds "Coinhive Blocker", "CoinBlock" (prefix search) and "NoMiners" (description)
Privacy -> Finds "Privacy Badger", "Privacy Pass", "Privacy Settings", "Google Privacy" (probably 4th or so) and "Blur" (summary + description + many users). Scores "Ghostery" on the first page but ranks it in the middle
firebu -> Finds "Firebug", "Firebug Autocompleter", "Firefinder for Firebug"
fireb -> Scores "Fire Drag" first, puts "Firebug" approximately 3rd or so
Menu Wizzard -> Finds "Menu Wizard" (fuzzy, typo) first, then "Add-ons Manager Context Menu" apparently because it matches good in the title and has many users
Frame Demolition -> Finds "Frame Demolition"
Demolition -> Finds only "Frame Demolition", same for "Demolation" (typo)
reStyle -> Finds "reStyle" and scores a few add-ons that match on "restore" next since the term is similar
MegaUpload DownloadHelper -> finds "MegaUpload DownloadHelper" first, scores "Video DownloadHelper" and "RadpidShare DownloadHelper" next. Doesn't find "Popup Blocker" anymore as currently happening on -prod
MegaUpload -> only finds "MegaUpload DownloadHelper" and nothing else
No Flash -> Scores "No Flash" first, then depending on users "Download Flash and Video", "YouTube Flash Video Player" and "YouTube Flash Player" (not necessarily in that order)
Disable Hello, Pocket & Reader+ -> finds "Disable Hello, Pocket & Reader+" first (yeay!), then scores "Reader", "Disable WebRTC" and "In My Pocket" next similarly to what's happening on -prod currently
Not working yet:
privacybadger -> "Privacy Badger" -> will probably need some kind of ngram filtering and analyzing (#591)
eyes -> 'decentraleyes' -> Not sure this should actually work, will probably need some more analyzing too (#591)
Not sure if it's specifically only because of these changes but #3248 is fixed.
This potentially fixes #7244, #6891, #6837, #6417, mozilla/addons#359.
Not sure if this fixes #mozilla/addons#567 but the results look much more promising and the amount of results doesn't explode here. I only have 2.8k add-ons for testing though so I'm not too sure.
And might be relevant to #6137.
This is a big step towards #2661, I doubt we can call this fixed though.
* Fix name tests
* Speed up ES tests, make scoring results more predictable by using only one shard and one replica
* Add tests
* Only test on new apiv3 based search.
* Fixup 'get_results'
* Remove debug print
* Isort imports
* Fix test settings, I'm blind.
* Adapt number of shards to what we define in settings_test
* Test search for grapple
* Remove property filtering in legacy api search, fix tests, fix flake8
* Add comment explaining shard config
* Fix tests again, add default platform, fix total counts again.
* Try to do some fixture cleanup, let's see...
* More test refactoring, make use of dfs-query-then-fetch during tests, allow us to test this later too via a waffle flag.
* Don't use waffle flag for legacy search, only for apiv3 searches.
* Fix unused import, fix usage of 'params'
* Use query-then-fetch in a regular addons-view tool, fix a few more tests to be able to handle the waffle-flag
* Create the dfs query then fetch flag only in ESTestCase, delete it properly. More serializer fixes
* precache the waffle flag for autocomplete tests too
* Minor cleanups
* Add docs, fix code style
* Add a todo
* Fix codestyle
2018-01-30 08:26:30 +03:00
|
|
|
from olympia.amo.tests import start_es_mocks, stop_es_mocks
|
2015-12-03 00:52:47 +03:00
|
|
|
from olympia.translations.hold import clean_translations
|
|
|
|
from olympia.users.models import UserProfile
|
2015-08-26 20:33:42 +03:00
|
|
|
|
2014-11-19 19:36:40 +03:00
|
|
|
|
2015-08-31 10:22:37 +03:00
|
|
|
@pytest.fixture(autouse=True)
|
|
|
|
def unpin_db(request):
|
|
|
|
"""Unpin the database from master in the current DB.
|
|
|
|
|
|
|
|
The `multidb` middleware pins the current thread to master for 15 seconds
|
|
|
|
after any POST request, which can lead to unexpected results for tests
|
|
|
|
of DB slave functionality."""
|
|
|
|
|
|
|
|
request.addfinalizer(pinning.unpin_this_thread)
|
|
|
|
|
|
|
|
|
2017-02-06 20:13:18 +03:00
|
|
|
@pytest.fixture(autouse=True)
|
|
|
|
def mock_elasticsearch():
|
|
|
|
"""Mock ElasticSearch in tests by default.
|
|
|
|
|
|
|
|
Tests that do need ES should inherit from ESTestCase, which will stop the
|
|
|
|
mock at setup time."""
|
|
|
|
start_es_mocks()
|
|
|
|
|
|
|
|
yield
|
|
|
|
|
|
|
|
stop_es_mocks()
|
|
|
|
|
|
|
|
|
2014-11-19 19:36:40 +03:00
|
|
|
@pytest.fixture(autouse=True)
|
|
|
|
def mock_inline_css(monkeypatch):
|
|
|
|
"""Mock jingo_minify.helpers.is_external: don't break on missing files.
|
|
|
|
|
|
|
|
When testing, we don't want nor need the bundled/minified css files, so
|
|
|
|
pretend that all the css files are external.
|
|
|
|
|
Merge pull request #5791 from mozilla/1573-implement-django-jinja
Loads of clean-ups along the way.
Fixes #1573
* Moves all helpers modules to templatetags.jinja_helpers - there are a few things in there that aren't jinja related so there'll be a follow-up to split that later
* Update Jinja2 to 2.9.6
* Add django-jinja 2.3.1
* move some jingo-minify helpers to olympia.lib to avoid any jingo imports
* unified |f and |fe helpers into one |format_html helper that always escapes everything unless the input is already |safe
* Move helpers, add requirements, fix imports.
* Give backend a proper name
* Port f and fe filters from jingo
* Make start porting reviews jinja helpers module, make first test pass
* Get a handful more tests working, get rid of get_env, more and more jingo calls.
* Disable extension matching, render .txt templates too, fixes more stats tests
* Fix csv_header template to produce some reasonable and csv parsable
* Set trim_blocks to be compatible with jingo
* Unify |f and |fe into one |format that simplifies escaping rules. Update tests.
I think having one unified |format filter is beneficial, especially
given that our current escaping rules were kinda complicated and allowed
for seriously dangerous combinations even with autoescape activated.
Now everything passed as an argument to |format is explicitly escaped
unless marked as |safe, there's no way around that anymore.
* Naive first version using |format instead of |f and |fe, might need more review.
* Fix format for unicode strings
* Rename |format to |format_html
* Move nl2br_xhtml to amo.jinja_helpers, use it in all our templates.
* Move datetime filter to our code temporarily, fix mocks, fix recursive error
* Move csrf call to django_jinja csrf_token tag, fix addons templatetags
* Fix |class_selected usage, fix render_with to get registered
* Remove hack from zadmin, fix zadmin templatetags
* Fix user delete photo test, don't rely on proper context updates that happen in order.
* Fix more nl2br tests
* Fix detail page escape, django escapes differently than jinja
* Remove context from edit_review_reply_form
* Fix date created format filter usage
* Remove templates setting user, fix can_review
* Fix collect_add_widget to not inherit the global context
* Remove happy unicode client
* Cleanup can_review, add tests
* Fix tests, django use different html character for double quote escaping
* Switch to firefox latest to check ui-tests
* Cleanup and fix do_truncate
* Remove jingo, copy jingo-minify helpers that depend on jingo to our lib for now
* Rename nl2br_xhtml -> nl2br
* Jinja 2.9 compat for reviews truncate filter
* Don't use TEMPLATE_DEBUG
* Set default value for q, happens on index page.
* Fix jinja2 formatting with newstyle gettext
2017-07-01 03:57:31 +03:00
|
|
|
Mocking this will prevent amo.templatetags.jinja_helpers.inline_css to
|
|
|
|
believe it should bundle the css.
|
2014-11-19 19:36:40 +03:00
|
|
|
|
|
|
|
"""
|
Merge pull request #5791 from mozilla/1573-implement-django-jinja
Loads of clean-ups along the way.
Fixes #1573
* Moves all helpers modules to templatetags.jinja_helpers - there are a few things in there that aren't jinja related so there'll be a follow-up to split that later
* Update Jinja2 to 2.9.6
* Add django-jinja 2.3.1
* move some jingo-minify helpers to olympia.lib to avoid any jingo imports
* unified |f and |fe helpers into one |format_html helper that always escapes everything unless the input is already |safe
* Move helpers, add requirements, fix imports.
* Give backend a proper name
* Port f and fe filters from jingo
* Make start porting reviews jinja helpers module, make first test pass
* Get a handful more tests working, get rid of get_env, more and more jingo calls.
* Disable extension matching, render .txt templates too, fixes more stats tests
* Fix csv_header template to produce some reasonable and csv parsable
* Set trim_blocks to be compatible with jingo
* Unify |f and |fe into one |format that simplifies escaping rules. Update tests.
I think having one unified |format filter is beneficial, especially
given that our current escaping rules were kinda complicated and allowed
for seriously dangerous combinations even with autoescape activated.
Now everything passed as an argument to |format is explicitly escaped
unless marked as |safe, there's no way around that anymore.
* Naive first version using |format instead of |f and |fe, might need more review.
* Fix format for unicode strings
* Rename |format to |format_html
* Move nl2br_xhtml to amo.jinja_helpers, use it in all our templates.
* Move datetime filter to our code temporarily, fix mocks, fix recursive error
* Move csrf call to django_jinja csrf_token tag, fix addons templatetags
* Fix |class_selected usage, fix render_with to get registered
* Remove hack from zadmin, fix zadmin templatetags
* Fix user delete photo test, don't rely on proper context updates that happen in order.
* Fix more nl2br tests
* Fix detail page escape, django escapes differently than jinja
* Remove context from edit_review_reply_form
* Fix date created format filter usage
* Remove templates setting user, fix can_review
* Fix collect_add_widget to not inherit the global context
* Remove happy unicode client
* Cleanup can_review, add tests
* Fix tests, django use different html character for double quote escaping
* Switch to firefox latest to check ui-tests
* Cleanup and fix do_truncate
* Remove jingo, copy jingo-minify helpers that depend on jingo to our lib for now
* Rename nl2br_xhtml -> nl2br
* Jinja 2.9 compat for reviews truncate filter
* Don't use TEMPLATE_DEBUG
* Set default value for q, happens on index page.
* Fix jinja2 formatting with newstyle gettext
2017-07-01 03:57:31 +03:00
|
|
|
from olympia.amo.templatetags import jinja_helpers
|
|
|
|
monkeypatch.setattr(jinja_helpers, 'is_external', lambda css: True)
|
2015-08-23 12:14:51 +03:00
|
|
|
|
|
|
|
|
|
|
|
def pytest_configure(config):
|
2017-05-30 08:30:46 +03:00
|
|
|
from olympia.amo.tests import prefix_indexes
|
2015-08-23 12:14:51 +03:00
|
|
|
prefix_indexes(config)
|
2015-08-26 20:33:42 +03:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(autouse=True, scope='session')
|
|
|
|
def instrument_jinja():
|
|
|
|
"""Make sure the "templates" list in a response is properly updated, even
|
|
|
|
though we're using Jinja2 and not the default django template engine."""
|
|
|
|
import jinja2
|
|
|
|
old_render = jinja2.Template.render
|
|
|
|
|
|
|
|
def instrumented_render(self, *args, **kwargs):
|
|
|
|
context = dict(*args, **kwargs)
|
|
|
|
test.signals.template_rendered.send(
|
|
|
|
sender=self, template=self, context=context)
|
|
|
|
return old_render(self, *args, **kwargs)
|
|
|
|
|
|
|
|
jinja2.Template.render = instrumented_render
|
|
|
|
|
|
|
|
|
2018-02-02 14:47:50 +03:00
|
|
|
def default_prefixer(settings):
|
2015-08-26 20:33:42 +03:00
|
|
|
"""Make sure each test starts with a default URL prefixer."""
|
|
|
|
request = http.HttpRequest()
|
|
|
|
request.META['SCRIPT_NAME'] = ''
|
|
|
|
prefixer = amo.urlresolvers.Prefixer(request)
|
|
|
|
prefixer.app = settings.DEFAULT_APP
|
|
|
|
prefixer.locale = settings.LANGUAGE_CODE
|
|
|
|
amo.urlresolvers.set_url_prefix(prefixer)
|
|
|
|
|
|
|
|
|
2018-02-02 14:47:50 +03:00
|
|
|
@pytest.yield_fixture(autouse=True)
|
|
|
|
def test_pre_setup(request, tmpdir, settings):
|
2015-08-26 20:33:42 +03:00
|
|
|
cache.clear()
|
|
|
|
# Override django-cache-machine caching.base.TIMEOUT because it's
|
|
|
|
# computed too early, before settings_test.py is imported.
|
|
|
|
caching.base.TIMEOUT = settings.CACHE_COUNT_TIMEOUT
|
|
|
|
|
|
|
|
translation.trans_real.deactivate()
|
|
|
|
# Django fails to clear this cache.
|
|
|
|
translation.trans_real._translations = {}
|
|
|
|
translation.trans_real.activate(settings.LANGUAGE_CODE)
|
|
|
|
|
2018-02-02 14:47:50 +03:00
|
|
|
settings.MEDIA_ROOT = str(tmpdir.mkdir('media'))
|
|
|
|
settings.TMP_PATH = str(tmpdir.mkdir('tmp'))
|
|
|
|
settings.NETAPP_STORAGE = settings.TMP_PATH
|
2018-01-04 12:12:56 +03:00
|
|
|
|
2018-02-02 14:47:50 +03:00
|
|
|
# Reset the prefixer and urlconf after updating media root
|
|
|
|
default_prefixer(settings)
|
|
|
|
|
|
|
|
from django.core.urlresolvers import clear_url_caches, set_urlconf
|
|
|
|
|
|
|
|
def _clear_urlconf():
|
|
|
|
clear_url_caches()
|
|
|
|
set_urlconf(None)
|
|
|
|
|
|
|
|
_clear_urlconf()
|
|
|
|
|
|
|
|
request.addfinalizer(_clear_urlconf)
|
|
|
|
|
|
|
|
yield
|
2015-08-26 20:33:42 +03:00
|
|
|
|
2017-03-06 19:19:34 +03:00
|
|
|
core.set_user(None)
|
2015-08-26 20:33:42 +03:00
|
|
|
clean_translations(None) # Make sure queued translations are removed.
|
|
|
|
|
|
|
|
# Make sure we revert everything we might have changed to prefixers.
|
|
|
|
amo.urlresolvers.clean_url_prefixes()
|
2015-09-10 13:32:58 +03:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def admin_group(db):
|
|
|
|
"""Create the Admins group."""
|
|
|
|
return Group.objects.create(name='Admins', rules='*:*')
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
2018-02-02 14:47:50 +03:00
|
|
|
def mozilla_user(admin_group, settings):
|
2015-09-10 13:32:58 +03:00
|
|
|
"""Create a "Mozilla User"."""
|
|
|
|
user = UserProfile.objects.create(pk=settings.TASK_USER_ID,
|
|
|
|
email='admin@mozilla.com',
|
|
|
|
username='admin')
|
|
|
|
user.save()
|
|
|
|
GroupUser.objects.create(user=user, group=admin_group)
|
|
|
|
return user
|