Fold jingo_minify into our code.
This will simplify the upgrade to Django 1.11 and allow us to keep this under control, upstream isn't properly maintained. * Cleanup jingo-minify, keep only parts we are using. Cleanup tests Also fix and simplify compress_assets to link to the correct /static/ folder. Originally we hardcoded to look for settings.STATIC_ROOT but that doesn't suffice since it's set to /site-static/ by default. We are now using hardcoding the source of our static files instead of guessing and mistakenly re-using STATIC_ROOT. This commit also removes verbosity as an argument and simply logs all errors immediately. To simplify testing it adds a --force parameter. To verify this works locally: * Change settings.STATIC_ROOT to `'/site-static/`' * Add `/site-static/` to the folders served by nginx (I'll open an issue on our nginx container to serve this by default too) * You may have to restart the nginx container after that for nginx to pick things up * Remove everything inside site-static folder * Run `make update_assets` Fixes #8532
This commit is contained in:
Christopher Grebs
Родитель
1000aacd45
Коммит
1bf79b82fa
|
|
@ -91,7 +91,7 @@ RUN DJANGO_SETTINGS_MODULE='settings_local' locale/compile-mo.sh locale
|
|||
# compile asssets
|
||||
RUN npm install \
|
||||
&& make -f Makefile-docker copy_node_js \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets --use-uuid -t \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py compress_assets \
|
||||
&& DJANGO_SETTINGS_MODULE='settings_local' python manage.py collectstatic --noinput
|
||||
|
||||
RUN rm -f settings_local.py settings_local.pyc
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ update_db:
|
|||
schematic src/olympia/migrations
|
||||
|
||||
update_assets:
|
||||
python manage.py compress_assets --use-uuid
|
||||
python manage.py compress_assets
|
||||
python manage.py collectstatic --noinput
|
||||
|
||||
update: update_deps update_db update_assets
|
||||
|
|
|
|||
16
conftest.py
16
conftest.py
|
|
@ -60,21 +60,6 @@ def mock_basket(settings):
|
|||
json={'status': 'ok', 'token': USER_TOKEN})
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_inline_css(monkeypatch):
|
||||
"""Mock jingo_minify.helpers.is_external: don't break on missing files.
|
||||
|
||||
When testing, we don't want nor need the bundled/minified css files, so
|
||||
pretend that all the css files are external.
|
||||
|
||||
Mocking this will prevent amo.templatetags.jinja_helpers.inline_css to
|
||||
believe it should bundle the css.
|
||||
|
||||
"""
|
||||
from olympia.amo.templatetags import jinja_helpers
|
||||
monkeypatch.setattr(jinja_helpers, 'is_external', lambda css: True)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
from olympia.amo.tests import prefix_indexes
|
||||
prefix_indexes(config)
|
||||
|
|
@ -118,6 +103,7 @@ def test_pre_setup(request, tmpdir, settings):
|
|||
|
||||
settings.MEDIA_ROOT = str(tmpdir.mkdir('media'))
|
||||
settings.TMP_PATH = str(tmpdir.mkdir('tmp'))
|
||||
settings.STATIC_ROOT = str(tmpdir.mkdir('site-static'))
|
||||
settings.NETAPP_STORAGE = settings.TMP_PATH
|
||||
|
||||
# Reset the prefixer and urlconf after updating media root
|
||||
|
|
|
|||
|
|
@ -217,9 +217,6 @@ isodate==0.6.0 \
|
|||
# jmespath is required by boto3
|
||||
jmespath==0.9.3 \
|
||||
--hash=sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63
|
||||
jingo_minify==0.7.0 \
|
||||
--hash=sha256:1427946e8d4082c909a98dbacfe1cd908cbdc83572003e1a0236da02b350e31f \
|
||||
--hash=sha256:f47ec7868467a1b270e115c301a4270137836ddb7b5f57552e65dff5c86aee05
|
||||
# kombu is required by celery
|
||||
kombu==4.2.1 \
|
||||
--hash=sha256:b274db3a4eacc4789aeb24e1de3e460586db7c4fc8610f7adcc7a3a1709a60af \
|
||||
|
|
|
|||
|
|
@ -0,0 +1,232 @@
|
|||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import subprocess
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.contrib.staticfiles.finders import find as find_static_path
|
||||
|
||||
from olympia.lib.jingo_minify_helpers import ensure_path_exists
|
||||
|
||||
|
||||
def run_command(command):
|
||||
"""Run a command and correctly poll the output and write that to stdout"""
|
||||
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
|
||||
while True:
|
||||
output = process.stdout.readline()
|
||||
if output == '' and process.poll() is not None:
|
||||
break
|
||||
if output:
|
||||
print(output.strip())
|
||||
return process.poll()
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = ('Compresses css and js assets defined in settings.MINIFY_BUNDLES')
|
||||
requires_model_validation = False
|
||||
|
||||
checked_hash = {}
|
||||
bundle_hashes = {}
|
||||
|
||||
missing_files = 0
|
||||
minify_skipped = 0
|
||||
|
||||
def add_arguments(self, parser):
|
||||
"""Handle command arguments."""
|
||||
parser.add_argument(
|
||||
'force', action='store_true',
|
||||
help='Ignores modified/created dates and forces compression.')
|
||||
|
||||
def generate_build_id(self):
|
||||
return uuid.uuid4().hex[:8]
|
||||
|
||||
def update_hashes(self):
|
||||
# Adds a time based hash on to the build id.
|
||||
self.build_id = '%s-%s' % (
|
||||
self.generate_build_id(), hex(int(time.time()))[2:])
|
||||
|
||||
build_id_file = os.path.realpath(
|
||||
os.path.join(settings.ROOT, 'build.py'))
|
||||
|
||||
with open(build_id_file, 'w') as f:
|
||||
f.write('BUILD_ID_CSS = "%s"\n' % self.build_id)
|
||||
f.write('BUILD_ID_JS = "%s"\n' % self.build_id)
|
||||
f.write('BUILD_ID_IMG = "%s"\n' % self.build_id)
|
||||
f.write('BUNDLE_HASHES = %s\n' % self.bundle_hashes)
|
||||
|
||||
def handle(self, **options):
|
||||
self.force_compress = options.get('force', False)
|
||||
|
||||
# This will loop through every bundle, and do the following:
|
||||
# - Concat all files into one
|
||||
# - Cache bust all images in CSS files
|
||||
# - Minify the concatted files
|
||||
for ftype, bundle in settings.MINIFY_BUNDLES.iteritems():
|
||||
for name, files in bundle.iteritems():
|
||||
# Set the paths to the files.
|
||||
concatted_file = os.path.join(
|
||||
settings.ROOT, 'static',
|
||||
ftype, '%s-all.%s' % (name, ftype,))
|
||||
compressed_file = os.path.join(
|
||||
settings.ROOT, 'static',
|
||||
ftype, '%s-min.%s' % (name, ftype,))
|
||||
|
||||
ensure_path_exists(concatted_file)
|
||||
ensure_path_exists(compressed_file)
|
||||
|
||||
files_all = []
|
||||
for fn in files:
|
||||
processed = self._preprocess_file(fn)
|
||||
# If the file can't be processed, we skip it.
|
||||
if processed is not None:
|
||||
files_all.append(processed)
|
||||
|
||||
# Concat all the files.
|
||||
tmp_concatted = '%s.tmp' % concatted_file
|
||||
if len(files_all) == 0:
|
||||
raise CommandError(
|
||||
'No input files specified in '
|
||||
'MINIFY_BUNDLES["%s"]["%s"] in settings.py!' %
|
||||
(ftype, name)
|
||||
)
|
||||
run_command('cat {files} > {tmp}'.format(
|
||||
files=' '.join(files_all),
|
||||
tmp=tmp_concatted
|
||||
))
|
||||
|
||||
# Cache bust individual images in the CSS.
|
||||
if ftype == 'css':
|
||||
bundle_hash = self._cachebust(tmp_concatted, name)
|
||||
self.bundle_hashes['%s:%s' % (ftype, name)] = bundle_hash
|
||||
|
||||
# Compresses the concatenations.
|
||||
is_changed = self._is_changed(concatted_file)
|
||||
self._clean_tmp(concatted_file)
|
||||
if is_changed or not os.path.isfile(compressed_file):
|
||||
self._minify(ftype, concatted_file, compressed_file)
|
||||
else:
|
||||
print(
|
||||
'File unchanged, skipping minification of %s' % (
|
||||
concatted_file))
|
||||
self.minify_skipped += 1
|
||||
|
||||
# Write out the hashes
|
||||
self.update_hashes()
|
||||
|
||||
if self.minify_skipped:
|
||||
print(
|
||||
'Unchanged files skipped for minification: %s' % (
|
||||
self.minify_skipped))
|
||||
|
||||
def _preprocess_file(self, filename):
|
||||
"""Preprocess files and return new filenames."""
|
||||
css_bin = filename.endswith('.less') and settings.LESS_BIN
|
||||
source = find_static_path(filename)
|
||||
target = source
|
||||
if css_bin:
|
||||
target = '%s.css' % source
|
||||
run_command('{lessc} {source} {target}'.format(
|
||||
lessc=css_bin,
|
||||
source=str(source),
|
||||
target=str(target)))
|
||||
return target
|
||||
|
||||
def _is_changed(self, concatted_file):
|
||||
"""Check if the file has been changed."""
|
||||
if self.force_compress:
|
||||
return True
|
||||
|
||||
tmp_concatted = '%s.tmp' % concatted_file
|
||||
file_exists = (
|
||||
os.path.exists(concatted_file) and
|
||||
os.path.getsize(concatted_file) == os.path.getsize(tmp_concatted))
|
||||
if file_exists:
|
||||
orig_hash = self._file_hash(concatted_file)
|
||||
temp_hash = self._file_hash(tmp_concatted)
|
||||
return orig_hash != temp_hash
|
||||
return True # Different filesize, so it was definitely changed
|
||||
|
||||
def _clean_tmp(self, concatted_file):
|
||||
"""Replace the old file with the temp file."""
|
||||
tmp_concatted = '%s.tmp' % concatted_file
|
||||
if os.path.exists(concatted_file):
|
||||
os.remove(concatted_file)
|
||||
os.rename(tmp_concatted, concatted_file)
|
||||
|
||||
def _cachebust(self, css_file, bundle_name):
|
||||
"""Cache bust images. Return a new bundle hash."""
|
||||
print('Cache busting images in %s' % re.sub('.tmp$', '', css_file))
|
||||
|
||||
if not os.path.exists(css_file):
|
||||
return
|
||||
|
||||
css_content = ''
|
||||
with open(css_file, 'r') as css_in:
|
||||
css_content = css_in.read()
|
||||
|
||||
def _parse(url):
|
||||
return self._cachebust_regex(url, css_file)
|
||||
|
||||
css_parsed = re.sub('url\(([^)]*?)\)', _parse, css_content)
|
||||
|
||||
with open(css_file, 'w') as css_out:
|
||||
css_out.write(css_parsed)
|
||||
|
||||
# Return bundle hash for cachebusting JS/CSS files.
|
||||
file_hash = hashlib.md5(css_parsed).hexdigest()[0:7]
|
||||
self.checked_hash[css_file] = file_hash
|
||||
|
||||
if self.missing_files:
|
||||
print(' - Error finding %s images' % (self.missing_files,))
|
||||
self.missing_files = 0
|
||||
|
||||
return file_hash
|
||||
|
||||
def _minify(self, ftype, file_in, file_out):
|
||||
"""Run the proper minifier on the file."""
|
||||
if ftype == 'js' and hasattr(settings, 'UGLIFY_BIN'):
|
||||
opts = {'method': 'UglifyJS', 'bin': settings.UGLIFY_BIN}
|
||||
run_command('{uglify} -v -o {target} {source} -m'.format(
|
||||
uglify=opts['bin'],
|
||||
target=file_out,
|
||||
source=file_in))
|
||||
elif ftype == 'css' and hasattr(settings, 'CLEANCSS_BIN'):
|
||||
opts = {'method': 'clean-css', 'bin': settings.CLEANCSS_BIN}
|
||||
run_command('{cleancss} -o {target} {source}'.format(
|
||||
cleancss=opts['bin'],
|
||||
target=file_out,
|
||||
source=file_in))
|
||||
|
||||
print('Minifying %s (using %s)' % (file_in, opts['method']))
|
||||
|
||||
def _file_hash(self, url):
|
||||
"""Open the file and get a hash of it."""
|
||||
if url in self.checked_hash:
|
||||
return self.checked_hash[url]
|
||||
|
||||
file_hash = ''
|
||||
try:
|
||||
with open(url) as f:
|
||||
file_hash = hashlib.md5(f.read()).hexdigest()[0:7]
|
||||
except IOError:
|
||||
self.missing_files += 1
|
||||
print(' - Could not find file %s' % url)
|
||||
|
||||
self.checked_hash[url] = file_hash
|
||||
return file_hash
|
||||
|
||||
def _cachebust_regex(self, img, parent):
|
||||
"""Run over the regex; img is the structural regex object."""
|
||||
url = img.group(1).strip('"\'')
|
||||
if url.startswith('data:') or url.startswith('http'):
|
||||
return 'url(%s)' % url
|
||||
|
||||
url = url.split('?')[0]
|
||||
full_url = os.path.join(
|
||||
settings.ROOT, os.path.dirname(parent), url)
|
||||
|
||||
return 'url(%s?%s)' % (url, self._file_hash(full_url))
|
||||
|
|
@ -2,7 +2,6 @@ import collections
|
|||
import json as jsonlib
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
|
||||
from operator import attrgetter
|
||||
from urlparse import urljoin
|
||||
|
|
@ -29,8 +28,7 @@ from olympia import amo
|
|||
from olympia.amo import urlresolvers, utils
|
||||
from olympia.constants.licenses import PERSONA_LICENSES_IDS
|
||||
from olympia.lib.jingo_minify_helpers import (
|
||||
_build_html, _get_compiled_css_url, get_css_urls, get_js_urls, get_path,
|
||||
is_external)
|
||||
_build_html, get_css_urls, get_js_urls)
|
||||
from olympia.lib.cache import cache_get_or_set, make_key
|
||||
|
||||
|
||||
|
|
@ -482,45 +480,6 @@ def _relative_to_absolute(url):
|
|||
return 'url(%s)' % url
|
||||
|
||||
|
||||
@library.global_function
|
||||
def inline_css(bundle, media=False, debug=None):
|
||||
"""
|
||||
If we are in debug mode, just output a single style tag for each css file.
|
||||
If we are not in debug mode, return a style that contains bundle-min.css.
|
||||
Forces a regular css() call for external URLs (no inline allowed).
|
||||
|
||||
Extracted from jingo-minify and re-registered, see:
|
||||
https://github.com/jsocol/jingo-minify/pull/41
|
||||
Added: turns relative links to absolute ones using STATIC_URL.
|
||||
"""
|
||||
if debug is None:
|
||||
debug = getattr(settings, 'DEBUG', False)
|
||||
|
||||
if debug:
|
||||
items = [_get_compiled_css_url(i)
|
||||
for i in settings.MINIFY_BUNDLES['css'][bundle]]
|
||||
else:
|
||||
items = ['css/%s-min.css' % bundle]
|
||||
|
||||
if not media:
|
||||
media = getattr(settings, 'CSS_MEDIA_DEFAULT', 'screen,projection,tv')
|
||||
|
||||
contents = []
|
||||
for css in items:
|
||||
if is_external(css):
|
||||
return _build_html([css], '<link rel="stylesheet" media="%s" '
|
||||
'href="%%s" />' % media)
|
||||
with open(get_path(css), 'r') as f:
|
||||
css_content = f.read()
|
||||
css_parsed = re.sub(r'url\(([^)]*?)\)',
|
||||
_relative_to_absolute,
|
||||
css_content)
|
||||
contents.append(css_parsed)
|
||||
|
||||
return _build_html(contents, '<style type="text/css" media="%s">%%s'
|
||||
'</style>' % media)
|
||||
|
||||
|
||||
# A (temporary?) copy of this is in services/utils.py. See bug 1055654.
|
||||
def user_media_path(what):
|
||||
"""Make it possible to override storage paths in settings.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
from importlib import import_module
|
||||
|
||||
from django.conf import settings
|
||||
|
|
@ -43,3 +44,59 @@ def test_cron_jobs_setting():
|
|||
for name, path in settings.CRON_JOBS.iteritems():
|
||||
module = import_module(path)
|
||||
getattr(module, name)
|
||||
|
||||
|
||||
@pytest.mark.static_assets
|
||||
def test_compress_assets_command_without_git():
|
||||
settings.MINIFY_BUNDLES = {
|
||||
'css': {'zamboni/css': ['css/legacy/main.css']}}
|
||||
|
||||
call_command('compress_assets')
|
||||
|
||||
build_id_file = os.path.realpath(os.path.join(settings.ROOT, 'build.py'))
|
||||
assert os.path.exists(build_id_file)
|
||||
|
||||
with open(build_id_file) as f:
|
||||
contents_before = f.read()
|
||||
|
||||
# Call command a second time. We should get a different build id, since it
|
||||
# depends on a uuid.
|
||||
call_command('compress_assets')
|
||||
with open(build_id_file) as f:
|
||||
contents_after = f.read()
|
||||
|
||||
assert contents_before != contents_after
|
||||
|
||||
|
||||
@pytest.mark.static_assets
|
||||
def test_compress_assets_correctly_fetches_static_images(settings, tmpdir):
|
||||
"""
|
||||
Make sure that `compress_assets` correctly fetches static assets
|
||||
such as icons and writes them correctly into our compressed
|
||||
and concatted files.
|
||||
|
||||
Refs https://github.com/mozilla/addons-server/issues/8760
|
||||
"""
|
||||
settings.MINIFY_BUNDLES = {
|
||||
'css': {'zamboni/css': ['css/legacy/main.css']}}
|
||||
|
||||
# Now run compress and collectstatic
|
||||
call_command('compress_assets', force=True)
|
||||
call_command('collectstatic', interactive=False)
|
||||
|
||||
css_all = os.path.join(
|
||||
settings.STATIC_ROOT, 'css', 'zamboni', 'css-all.css')
|
||||
|
||||
css_min = os.path.join(
|
||||
settings.STATIC_ROOT, 'css', 'zamboni', 'css-min.css')
|
||||
|
||||
with open(css_all, 'rb') as fobj:
|
||||
expected = 'background-image: url(../../img/icons/stars.png'
|
||||
assert expected in fobj.read()
|
||||
|
||||
# Compressed doesn't have any whitespace between `background-image:` and
|
||||
# the url and the path is slightly different
|
||||
with open(css_min, 'rb') as fobj:
|
||||
data = fobj.read()
|
||||
assert 'background-image:url(' in data
|
||||
assert 'img/icons/stars.png' in data
|
||||
|
|
|
|||
|
|
@ -3,12 +3,10 @@ import subprocess
|
|||
import time
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.staticfiles.finders import find as find_static_path
|
||||
|
||||
import jinja2
|
||||
|
||||
from jingo_minify.utils import get_media_url, get_path
|
||||
|
||||
|
||||
try:
|
||||
from build import BUILD_ID_CSS, BUILD_ID_JS, BUILD_ID_IMG, BUNDLE_HASHES
|
||||
except ImportError:
|
||||
|
|
@ -29,14 +27,14 @@ def _get_item_path(item):
|
|||
"""
|
||||
if is_external(item):
|
||||
return item
|
||||
return get_media_url() + item
|
||||
return settings.STATIC_URL + item
|
||||
|
||||
|
||||
def _get_mtime(item):
|
||||
"""Get a last-changed timestamp for development."""
|
||||
if item.startswith(('//', 'http://', 'https://')):
|
||||
return int(time.time())
|
||||
return int(os.path.getmtime(get_path(item)))
|
||||
return int(os.path.getmtime(find_static_path(item)))
|
||||
|
||||
|
||||
def _build_html(items, wrapping):
|
||||
|
|
@ -46,6 +44,17 @@ def _build_html(items, wrapping):
|
|||
return jinja2.Markup('\n'.join((wrapping % item for item in items)))
|
||||
|
||||
|
||||
def ensure_path_exists(path):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path))
|
||||
except OSError as e:
|
||||
# If the directory already exists, that is fine. Otherwise re-raise.
|
||||
if e.errno != os.errno.EEXIST:
|
||||
raise
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def get_js_urls(bundle, debug=None):
|
||||
"""
|
||||
Fetch URLs for the JS files in the requested bundle.
|
||||
|
|
@ -72,19 +81,6 @@ def get_js_urls(bundle, debug=None):
|
|||
return (_get_item_path('js/%s-min.js?build=%s' % (bundle, build_id,)),)
|
||||
|
||||
|
||||
def _get_compiled_css_url(item):
|
||||
"""
|
||||
Compresses a preprocess file and returns its relative compressed URL.
|
||||
|
||||
:param item:
|
||||
Name of the less file to compress into css.
|
||||
"""
|
||||
if item.endswith('.less') and getattr(settings, 'LESS_PREPROCESS', False):
|
||||
compile_css(item)
|
||||
return item + '.css'
|
||||
return item
|
||||
|
||||
|
||||
def get_css_urls(bundle, debug=None):
|
||||
"""
|
||||
Fetch URLs for the CSS files in the requested bundle.
|
||||
|
|
@ -102,9 +98,11 @@ def get_css_urls(bundle, debug=None):
|
|||
if debug:
|
||||
items = []
|
||||
for item in settings.MINIFY_BUNDLES['css'][bundle]:
|
||||
if ((item.endswith('.less') and
|
||||
getattr(settings, 'LESS_PREPROCESS', False)) or
|
||||
item.endswith(('.sass', '.scss', '.styl'))):
|
||||
should_compile = (
|
||||
item.endswith('.less') and
|
||||
getattr(settings, 'LESS_PREPROCESS', False))
|
||||
|
||||
if should_compile:
|
||||
compile_css(item)
|
||||
items.append('%s.css' % item)
|
||||
else:
|
||||
|
|
@ -121,26 +119,18 @@ def get_css_urls(bundle, debug=None):
|
|||
(bundle, build_id)),)
|
||||
|
||||
|
||||
def ensure_path_exists(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as e:
|
||||
# If the directory already exists, that is fine. Otherwise re-raise.
|
||||
if e.errno != os.errno.EEXIST:
|
||||
raise
|
||||
|
||||
|
||||
def compile_css(item):
|
||||
path_src = get_path(item)
|
||||
path_dst = get_path('%s.css' % item)
|
||||
path_src = find_static_path(item)
|
||||
path_dst = os.path.join(
|
||||
settings.ROOT, 'static', '%s.css' % item)
|
||||
|
||||
updated_src = os.path.getmtime(get_path(item))
|
||||
updated_css = 0 # If the file doesn't exist, force a refresh.
|
||||
updated_src = os.path.getmtime(find_static_path(item))
|
||||
updated_dst = 0 # If the file doesn't exist, force a refresh.
|
||||
if os.path.exists(path_dst):
|
||||
updated_css = os.path.getmtime(path_dst)
|
||||
updated_dst = os.path.getmtime(path_dst)
|
||||
|
||||
# Is the uncompiled version newer? Then recompile!
|
||||
if not updated_css or updated_src > updated_css:
|
||||
if not updated_dst or updated_src > updated_dst:
|
||||
ensure_path_exists(os.path.dirname(path_dst))
|
||||
if item.endswith('.less'):
|
||||
with open(path_dst, 'w') as output:
|
||||
|
|
|
|||
|
|
@ -520,7 +520,6 @@ INSTALLED_APPS = (
|
|||
'raven.contrib.django',
|
||||
'rest_framework',
|
||||
'waffle',
|
||||
'jingo_minify',
|
||||
'django_jinja',
|
||||
'puente',
|
||||
|
||||
|
|
@ -1694,8 +1693,6 @@ DEV_AGREEMENT_LAST_UPDATED = None
|
|||
# In production we do not want to allow this.
|
||||
ALLOW_SELF_REVIEWS = False
|
||||
|
||||
# This saves us when we upgrade jingo-minify (jsocol/jingo-minify@916b054c).
|
||||
JINGO_MINIFY_USE_STATIC = True
|
||||
|
||||
# Allow URL style format override. eg. "?format=json"
|
||||
URL_FORMAT_OVERRIDE = 'format'
|
||||
|
|
@ -1719,11 +1716,11 @@ CDN_HOST = ''
|
|||
# Static
|
||||
STATIC_ROOT = path('site-static')
|
||||
STATIC_URL = '/static/'
|
||||
JINGO_MINIFY_ROOT = path('static')
|
||||
|
||||
STATICFILES_DIRS = (
|
||||
path('static'),
|
||||
JINGO_MINIFY_ROOT
|
||||
)
|
||||
|
||||
NETAPP_STORAGE = TMP_PATH
|
||||
GUARDED_ADDONS_PATH = ROOT + '/guarded-addons'
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,261 @@
|
|||
import os
|
||||
|
||||
from django.conf import settings
|
||||
from django.test.utils import override_settings
|
||||
|
||||
import mock
|
||||
|
||||
from olympia.amo.utils import from_string
|
||||
|
||||
try:
|
||||
from build import BUILD_ID_CSS, BUILD_ID_JS, BUILD_ID_IMG, BUNDLE_HASHES
|
||||
except ImportError:
|
||||
BUILD_ID_CSS = BUILD_ID_JS = BUILD_ID_IMG = 'dev'
|
||||
BUNDLE_HASHES = {}
|
||||
|
||||
|
||||
TEST_MINIFY_BUNDLES = {
|
||||
'css': {
|
||||
'common': ['css/test.css'],
|
||||
'common_url': ['http://example.com/test.css'],
|
||||
'common_protocol_less_url': ['//example.com/test.css'],
|
||||
'common_bundle': ['css/test.css', 'http://example.com/test.css',
|
||||
'//example.com/test.css',
|
||||
'https://example.com/test.css'],
|
||||
'compiled': ['css/plain.css', 'css/less.less']
|
||||
},
|
||||
'js': {
|
||||
'common': ['js/test.js'],
|
||||
'common_url': ['http://example.com/test.js'],
|
||||
'common_protocol_less_url': ['//example.com/test.js'],
|
||||
'common_bundle': ['js/test.js', 'http://example.com/test.js',
|
||||
'//example.com/test.js',
|
||||
'https://example.com/test.js'],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@override_settings(MINIFY_BUNDLES=TEST_MINIFY_BUNDLES)
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.time.time')
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.os.path.getmtime')
|
||||
def test_js_helper(getmtime, time):
|
||||
"""
|
||||
Given the js() tag if we return the assets that make up that bundle
|
||||
as defined in settings.MINIFY_BUNDLES.
|
||||
|
||||
If we're not in debug mode, we just return a minified url
|
||||
"""
|
||||
getmtime.return_value = 1
|
||||
time.return_value = 1
|
||||
|
||||
template = from_string('{{ js("common", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = '\n'.join([
|
||||
'<script src="%s?build=1"></script>' % (settings.STATIC_URL + j)
|
||||
for j in settings.MINIFY_BUNDLES['js']['common']])
|
||||
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ js("common", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<script src="%sjs/common-min.js?build=%s"></script>' %
|
||||
(settings.STATIC_URL, BUILD_ID_JS))
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ js("common_url", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = '<script src="http://example.com/test.js?build=1"></script>'
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ js("common_url", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<script src="%sjs/common_url-min.js?build=%s"></script>' %
|
||||
(settings.STATIC_URL, BUILD_ID_JS))
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ js("common_protocol_less_url", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
assert rendered == '<script src="//example.com/test.js?build=1"></script>'
|
||||
|
||||
template = from_string('{{ js("common_protocol_less_url", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<script src="%sjs/common_protocol_less_url-min.js?build=%s"></script>'
|
||||
% (settings.STATIC_URL, BUILD_ID_JS))
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ js("common_bundle", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
assert (
|
||||
rendered == (
|
||||
'<script src="%sjs/test.js?build=1"></script>\n'
|
||||
'<script src="http://example.com/test.js?build=1"></script>\n'
|
||||
'<script src="//example.com/test.js?build=1"></script>\n'
|
||||
'<script src="https://example.com/test.js?build=1"></script>'
|
||||
% settings.STATIC_URL))
|
||||
|
||||
template = from_string('{{ js("common_bundle", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
assert (
|
||||
rendered ==
|
||||
'<script src="%sjs/common_bundle-min.js?build=%s"></script>' %
|
||||
(settings.STATIC_URL, BUILD_ID_JS))
|
||||
|
||||
|
||||
@override_settings(MINIFY_BUNDLES=TEST_MINIFY_BUNDLES)
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.time.time')
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.os.path.getmtime')
|
||||
def test_css_helper(getmtime, time):
|
||||
"""
|
||||
Given the css() tag if we return the assets that make up that bundle
|
||||
as defined in settings.MINIFY_BUNDLES.
|
||||
|
||||
If we're not in debug mode, we just return a minified url
|
||||
"""
|
||||
getmtime.return_value = 1
|
||||
time.return_value = 1
|
||||
|
||||
template = from_string('{{ css("common", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = "\n".join([
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="%s?build=1" />' % (settings.STATIC_URL + j)
|
||||
for j in settings.MINIFY_BUNDLES['css']['common']
|
||||
])
|
||||
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ css("common", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="%scss/common-min.css?build=%s" />'
|
||||
% (settings.STATIC_URL, BUILD_ID_CSS))
|
||||
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ css("common_url", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="http://example.com/test.css?build=1" />')
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ css("common_url", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="%scss/common_url-min.css?build=%s" />'
|
||||
% (settings.STATIC_URL, BUILD_ID_CSS))
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ css("common_protocol_less_url", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
assert (
|
||||
rendered == (
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="//example.com/test.css?build=1" />'))
|
||||
|
||||
template = from_string(
|
||||
'{{ css("common_protocol_less_url", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = (
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="%scss/common_protocol_less_url-min.css?build=%s" />'
|
||||
% (settings.STATIC_URL, BUILD_ID_CSS))
|
||||
|
||||
assert rendered == expected
|
||||
|
||||
template = from_string('{{ css("common_bundle", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
assert (
|
||||
rendered ==
|
||||
'<link rel="stylesheet" media="all" href="/static/css/test.css?build=1" />\n' # noqa
|
||||
'<link rel="stylesheet" media="all" href="http://example.com/test.css?build=1" />\n' # noqa
|
||||
'<link rel="stylesheet" media="all" href="//example.com/test.css?build=1" />\n' # noqa
|
||||
'<link rel="stylesheet" media="all" href="https://example.com/test.css?build=1" />') # noqa
|
||||
|
||||
template = from_string('{{ css("common_bundle", debug=False) }}')
|
||||
rendered = template.render()
|
||||
|
||||
assert (
|
||||
rendered ==
|
||||
'<link rel="stylesheet" media="all" '
|
||||
'href="%scss/common_bundle-min.css?build=%s" />' %
|
||||
(settings.STATIC_URL, BUILD_ID_CSS))
|
||||
|
||||
|
||||
@override_settings(STATIC_URL='http://example.com/static/',
|
||||
MEDIA_URL='http://example.com/media/',
|
||||
MINIFY_BUNDLES=TEST_MINIFY_BUNDLES)
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.time.time')
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.os.path.getmtime')
|
||||
def test_css(getmtime, time):
|
||||
getmtime.return_value = 1
|
||||
time.return_value = 1
|
||||
|
||||
template = from_string('{{ css("common", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = "\n".join(
|
||||
['<link rel="stylesheet" media="all" '
|
||||
'href="%s?build=1" />' % (settings.STATIC_URL + j)
|
||||
for j in settings.MINIFY_BUNDLES['css']['common']])
|
||||
|
||||
assert rendered == expected
|
||||
|
||||
|
||||
@override_settings(MINIFY_BUNDLES={
|
||||
'css': {'compiled': ['css/impala/buttons.less']}})
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.os.path.getmtime')
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.subprocess')
|
||||
@mock.patch('__builtin__.open', spec=True)
|
||||
def test_compiled_css(open_mock, subprocess_mock, getmtime_mock):
|
||||
getmtime_mock.side_effect = [
|
||||
# The first call is for the source
|
||||
1531144805.1225898,
|
||||
# The second call is for the destination
|
||||
1530885814.6340182]
|
||||
|
||||
from_string('{{ css("compiled", debug=True) }}")').render()
|
||||
|
||||
source = os.path.realpath(os.path.join(
|
||||
settings.ROOT, 'static/css/impala/buttons.less'))
|
||||
|
||||
assert subprocess_mock.Popen.mock_calls == [
|
||||
mock.call([settings.LESS_BIN, source], stdout=mock.ANY)]
|
||||
|
||||
|
||||
@override_settings(STATIC_URL='http://example.com/static/',
|
||||
MEDIA_URL='http://example.com/media/')
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.time.time')
|
||||
@mock.patch('olympia.lib.jingo_minify_helpers.os.path.getmtime')
|
||||
def test_js(getmtime, time):
|
||||
getmtime.return_value = 1
|
||||
time.return_value = 1
|
||||
|
||||
template = from_string('{{ js("common", debug=True) }}')
|
||||
rendered = template.render()
|
||||
|
||||
expected = "\n".join(
|
||||
['<script src="%s?build=1"></script>' % (settings.STATIC_URL + j)
|
||||
for j in settings.MINIFY_BUNDLES['js']['common']])
|
||||
|
||||
assert rendered == expected
|
||||
15
tox.ini
15
tox.ini
|
|
@ -18,39 +18,39 @@ whitelist_externals =
|
|||
[testenv:es]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies
|
||||
pytest -m "es_tests and not needs_locales_compilation" --ignore=tests/ui/ -v {posargs}
|
||||
pytest -m "es_tests and not needs_locales_compilation and not static_assets" --ignore=tests/ui/ -v src/olympia/{posargs}
|
||||
|
||||
[testenv:addons]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/addons/ {posargs}
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/addons/ {posargs}
|
||||
|
||||
[testenv:devhub]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies install_node_dependencies
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/devhub/ {posargs}
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/devhub/ {posargs}
|
||||
|
||||
[testenv:reviewers]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/reviewers/ {posargs}
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/reviewers/ {posargs}
|
||||
|
||||
[testenv:amo-locales-and-signing]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies install_node_dependencies
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/amo/ src/olympia/lib/crypto/ src/olympia/signing {posargs}
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/amo/ src/olympia/lib/crypto/ src/olympia/signing {posargs}
|
||||
bash {toxinidir}/locale/compile-mo.sh {toxinidir}/locale/
|
||||
pytest -n 2 -m 'needs_locales_compilation' -v src/olympia/ {posargs}
|
||||
|
||||
[testenv:users-and-accounts]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/users/ src/olympia/accounts/ {posargs}
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/users/ src/olympia/accounts/ {posargs}
|
||||
|
||||
[testenv:main]
|
||||
commands =
|
||||
make -f Makefile-docker install_python_test_dependencies install_node_dependencies
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation' -v src/olympia/ --ignore src/olympia/addons/ --ignore src/olympia/devhub/ --ignore src/olympia/reviewers/ --ignore src/olympia/amo/ --ignore src/olympia/users/ --ignore src/olympia/accounts/ --ignore src/olympia/lib/crypto --ignore src/olympia/signing {posargs}
|
||||
pytest -n 2 -m 'not es_tests and not needs_locales_compilation and not static_assets' -v src/olympia/ --ignore src/olympia/addons/ --ignore src/olympia/devhub/ --ignore src/olympia/reviewers/ --ignore src/olympia/amo/ --ignore src/olympia/users/ --ignore src/olympia/accounts/ --ignore src/olympia/lib/crypto --ignore src/olympia/signing {posargs}
|
||||
|
||||
[testenv:ui-tests]
|
||||
commands =
|
||||
|
|
@ -61,6 +61,7 @@ commands =
|
|||
[testenv:assets]
|
||||
commands =
|
||||
make -f Makefile-docker update_deps
|
||||
pytest -m "static_assets" --ignore=tests/ui/ -v src/olympia/ {posargs}
|
||||
make -f Makefile-docker update_assets
|
||||
|
||||
[testenv:codestyle]
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче