mozilla
/
commonplace
зеркало из https://github.com/mozilla/commonplace.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Rewrite build tools to be more async and centralized

This commit is contained in:
Matt Basta 2013-08-13 17:32:51 -07:00
Родитель 764ba65e11
Коммит 3efa8fbbc5
8 изменённых файлов: 341 добавлений и 278 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
.gitignore поставляемый
Просмотреть файл

@ -23,9 +23,9 @@ build
dist
src/locales/*.js
src/media/css/include.css
src/media/js/include.js
src/media/include.css
src/media/include.js
src/media/js/include.js
src/templates.js
node_modules
npm-debug.log

8
bin/commonplace
Просмотреть файл

@ -10,8 +10,8 @@ function help() {
' update This command will update commonplace modules.',
' extract_strings This command will extract strings into a `.pot` file.',
' langpacks This command will generate langpacks out of `.po` files.',
' clean Run this to clean up static files, templates and langpacks.',
' raw_includes Combine all the CSS and JS assets',
' clean Clean up static files, templates, and langpacks.',
' compile Generate CSS from stylus and JS from templates.',
' includes Combine and minify all the CSS and JS assets.'
].join('\n'));
}
@ -43,8 +43,8 @@ switch (argv[0]) {
case 'langpacks':
commonplace.generate_langpacks();
break;
case 'raw_includes':
commonplace.build_includes(true);
case 'compile':
commonplace.compile();
break;
case 'includes':
commonplace.build_includes();

194
bin/damper
Просмотреть файл

@ -2,13 +2,13 @@
var fs = require('fs');
var http = require('http');
var path = require('path');
var stylus = require('stylus');
var utils = require('../lib/utils.js');
var info = require('../lib/info.js');
var opts = utils.opts;
var glob = utils.glob;
var src_dir = require('../lib/info.js').src_dir();
process.title = 'damper';
@ -35,150 +35,110 @@ info.check_version(
}
);
var opts = opts(process.argv.slice(2),
{host: '0.0.0.0', port: '8675', compile: false});
var opts = utils.opts(process.argv.slice(2),
{host: '0.0.0.0', port: '8675'});
if (!opts.compile) {
// Here's the local server.
http.createServer(function(request, response) {
// Here's the local server.
http.createServer(function(request, response) {
var now = new Date();
var now = new Date();
console.log(
'[' + now.getHours() + ':' + now.getMinutes() + ':' + now.getSeconds() + '] ' +
request.url);
console.log(
'[' + now.getHours() + ':' + now.getMinutes() + ':' + now.getSeconds() + '] ' +
request.url);
function writeIndex() {
fs.readFile(src_dir + '/index.html', function(error, content) {
// We'll assume that you don't delete index.html.
response.writeHead(200, {'Content-Type': 'text/html'});
response.end(content, 'utf-8');
});
}
if (request.url == '/') {
return writeIndex();
}
var qindex;
if ((qindex = request.url.indexOf('?')) !== -1) {
request.url = request.url.substr(0, qindex);
}
var filePath = src_dir + request.url;
fs.exists(filePath, function(exists) {
if (exists && !fs.statSync(filePath).isDirectory()) {
fs.readFile(filePath, function(error, content) {
if (error) {
response.writeHead(500);
response.end();
console.error(error);
} else {
var dot = request.url.lastIndexOf('.');
if (dot > -1) {
var extension = request.url.substr(dot + 1);
response.writeHead(200, {'Content-Type': mimes[extension]});
}
response.end(content, 'utf-8');
}
});
} else {
writeIndex();
}
function writeIndex() {
fs.readFile(path.resolve(src_dir, 'index.html'), function(error, content) {
// We'll assume that you don't delete index.html.
response.writeHead(200, {'Content-Type': 'text/html'});
response.end(content, 'utf-8');
});
}
}).listen(opts.port, opts.host);
if (request.url == '/') {
return writeIndex();
}
console.log('Server running at http://' + opts.host + ':' + opts.port);
} else {
console.log('Starting compilation...');
}
var qindex;
if ((qindex = request.url.indexOf('?')) !== -1) {
request.url = request.url.substr(0, qindex);
}
var url_path = request.url;
if (url_path[0] === '/') {
url_path = url_path.substr(1);
}
var filePath = path.resolve(src_dir, url_path);
fs.exists(filePath, function(exists) {
if (!exists || fs.statSync(filePath).isDirectory()) {
writeIndex();
return;
}
fs.readFile(filePath, function(err, content) {
if (err) {
response.writeHead(500);
response.end();
console.error(err);
return;
}
var dot = request.url.lastIndexOf('.');
if (dot > -1) {
var extension = request.url.substr(dot + 1);
response.writeHead(200, {'Content-Type': mimes[extension]});
}
response.end(content, 'utf-8');
});
});
}).listen(opts.port, opts.host);
console.log('Server running at http://' + opts.host + ':' + opts.port);
var watched_filepaths = [];
function runCommand(command, filepath) {
switch (command) {
case 'stylus':
filepath = filepath || opts.path;
var filepathDir = filepath.split('/').slice(0, -1).join('/');
fs.readFile(filepath, function (err, data) {
data = data.toString();
if (err) {
console.error(filepath + ' not found!');
return;
}
stylus(data)
.set('filename', filepath + '.css')
.set('include css', true)
.include(filepathDir)
.render(function(err, css) {
fs.writeFileSync(filepath + '.css', css);
if (err) console.error(err);
});
require('../lib/build').stylus(filepath, function(err, css) {
fs.writeFileSync(filepath + '.css', css);
if (err) console.error(err);
});
break;
case 'nunjucks':
console.log('Recompiling templates...');
require('../lib/compile').process(
src_dir + '/templates',
src_dir + '/templates.js'
);
require('../lib/commonplace').compile({silent: true, only: ['nunjucks']});
break;
}
}
function watch(globpath, ext, command) {
var cb = function(err, filepaths) {
filepaths.forEach(function(filepath) {
watched_filepaths.push(filepath);
if (command == 'stylus') {
fs.exists(filepath, function(exists) {
if (exists) {
runCommand(command, filepath);
}
});
}
fs.exists(globpath, function(exists) {
if (!exists) {
console.warn('Skipping absent path: ' + globpath);
return;
}
// If we're compiling, we don't want to watch the files.
if (opts.compile) {
return;
var count = 0;
utils.globEach(globpath, ext, function(filepath) {
count++;
if (command == 'stylus') {
runCommand(command, filepath);
}
fs.watchFile(filepath, {interval: 250}, function(curr, prev) {
if (curr.mtime.valueOf() != prev.mtime.valueOf() ||
curr.ctime.valueOf() != prev.ctime.valueOf()) {
if (curr.mtime.valueOf() !== prev.mtime.valueOf() ||
curr.ctime.valueOf() !== prev.ctime.valueOf()) {
console.warn('> ' + filepath + ' changed.');
runCommand(command, filepath);
}
});
}, function() {
console.log('Watching ' + count + ' `' + ext + '` files.');
});
if (filepaths.length > 1 && !opts.compile) {
console.log('Watching ' + filepaths.length + ' ' + ext + ' files.');
}
};
if (!fs.existsSync(globpath)) {
console.warn('Skipping non-existing path: ' + globpath);
return;
}
if (globpath.substr(1).indexOf('.') > -1) {
cb(null, [globpath]);
} else {
glob(globpath, ext, cb);
}
}
if (opts.compile && opts.compile !== true) {
runCommand(opts.compile);
} else {
runCommand('nunjucks');
watch(src_dir + '/media/css', 'styl', 'stylus');
watch(src_dir + '/templates', 'html', 'nunjucks');
// When the builder is updated, recompile the templates.
watch(src_dir + '/media/js/builder.js', null, 'nunjucks');
});
}
runCommand('nunjucks');
watch(path.resolve(src_dir, 'media/css'), 'styl', 'stylus');
watch(path.resolve(src_dir, 'templates'), 'html', 'nunjucks');

0
lib/amd/amd.js → lib/assets/amd.js
Просмотреть файл

191
lib/build.js
Просмотреть файл

@ -1,5 +1,6 @@
var fs = require('fs');
var path = require('path');
var utils = require('./utils.js');
var blacklist = [
@ -9,72 +10,162 @@ var blacklist = [
'settings_travis.js',
'splash.styl.css',
'templates.js', // Generated dynamically.
'include.js',
'include.css',
];
function concatJS(assetsPath, includes, output_file) {
function concatJS(src_dir, callback) {
var output = '';
includes.forEach(function(include) {
include = assetsPath + '/' + include;
if (include.substr(-3) == '.js') {
output += fs.readFileSync(include).toString();
} else {
utils.globSync(include, '.js', function(err, results) {
results.forEach(function(file) {
if (blacklist.indexOf(path.basename(file)) === -1) {
output += fs.readFileSync(file).toString() + '\n';
}
});
});
}
// Render the HTML for the templates.
compileHTML(src_dir, function(html_data) {
output += html_data;
// Now include the rest of the JS.
utils.globEach(
path.resolve(src_dir, 'media/js'),
'.js',
function(file) {
if (blacklist.indexOf(path.basename(file)) !== -1) return;
output += fs.readFileSync(file) + '\n';
}, function() {
callback(output);
}
);
});
var include = fs.readFileSync(assetsPath + '/' + output_file).toString();
output = include.replace(/'replace me'/, function(){
return output;
});
fs.writeFileSync(assetsPath + '/' + output_file, output);
}
function concatCSS(assetsPath, includes, output_file) {
var output = '';
var match;
var css_pattern = /href="(\/media\/css\/.+\.styl\.css)"/g;
function compileStylus(source_path, callback) {
var stylus = require('stylus');
fs.readFile(source_path, function(err, data) {
if (err) {
console.error('Could not read stylus file: ' + source_path);
callback(err);
return;
}
includes.forEach(function(include) {
var include_path = [assetsPath, '/', include].join('');
var include_data = fs.readFileSync(include_path).toString();
if (include.substr(-5) === '.html') {
while (match = css_pattern.exec(include_data)) {
if (blacklist.indexOf(path.basename(match[1])) === -1) {
output += fs.readFileSync(assetsPath + match[1]).toString();
stylus(data.toString())
.set('filename', source_path + '.css')
.set('include css', true)
.include(path.dirname(source_path))
.render(function(err, css) {
if (err) {
console.error('Error compiling stylus file: ' + source_path);
callback(err);
return;
}
callback(null, css);
});
});
}
function concatCSS(src_dir, callback) {
var output = '';
var css_pattern = /href="(\/media\/css\/.+\.styl\.css)"/g;
var url_pattern = /url\(([^)]+)\)/g;
function fix_urls(data) {
return data.replace(url_pattern, function(match, url, offset, string) {
url = url.replace(/"|'/g, '');
if (url.search(/(https?|data):|\/\//) === 0) {
return ['url(', url, ')'].join('');
}
var timestamp = new Date().getTime();
if (url.indexOf('#') !== -1) {
var split = url.split('#');
return ['url(', split[0], '?', timestamp, '#', split[1], ')'].join('');
} else {
return ['url(', url, '?', timestamp, ')'].join('');
}
});
}
var index_html = fs.readFile(path.resolve(src_dir, 'index.html'), function(err, data) {
if (err) {
console.error('Could not read `index.html`.', err);
return;
}
var match;
var files = [];
data = data.toString();
while (match = css_pattern.exec(data)) {
if (blacklist.indexOf(path.basename(match[1])) === -1) {
files.push(path.resolve(src_dir + match[1]));
}
}
});
var url_pattern = /url\(([^)]+)\)/g;
output = output.replace(url_pattern, function(match, url, offset, string) {
url = url.replace(/"|'/g, '');
if (url.search(/(https?|data):|\/\//) === 0) {
return ['url(', url, ')'].join('');
}
var remaining = files.length;
files.forEach(function(v) {
v = v.replace('.styl.css', '.styl');
compileStylus(v, function(err, data) {
remaining--;
if (err) {
console.warn(err);
var timestamp = new Date().getTime();
if (url.indexOf('#') !== -1) {
var split = url.split('#');
return ['url(', split[0], '?', timestamp, '#', split[1], ')'].join('');
} else {
return ['url(', url, '?', timestamp, ')'].join('');
}
if (!remaining) callback(output);
return;
}
output += fix_urls(data + '\n');
if (!remaining) callback(output);
});
});
});
fs.writeFileSync(assetsPath + '/' + output_file, output);
}
function build(assetsPath) {
console.log('Building assets...');
concatJS(assetsPath, ['media/js', 'templates.js'], 'media/include.js');
concatCSS(assetsPath, ['index.html'], 'media/include.css');
function compileHTML(src_dir, callback) {
var compiler = require('nunjucks').compiler;
var parser = require('nunjucks').parser;
var extensions = require('./deferparser').extensions || [];
var template_dir = path.resolve(src_dir, 'templates');
if (template_dir.substr(-1) !== '/') {
template_dir += '/';
}
var template_data = [];
utils.globEach(template_dir, '.html', function(template) {
var name = template.replace(template_dir, '');
var output = 'templates["' + name + '"] = (function() {';
try {
var src = fs.readFileSync(template, 'utf-8');
var cinst = new compiler.Compiler(extensions);
// Parse
var parsed = parser.parse(src, extensions);
// Compile
cinst.compile(parsed);
// Output
output += cinst.getCode();
} catch(e) {
output += [
'return {root: function() {',
'throw new Error("' + name + ' failed to compile. Check the damper for details.");',
'}}'
].join('\n');
console.error(e);
}
template_data.push(output + '})();\n');
}, function() {
callback(
'(function() {' +
'var templates = {};\n' +
template_data.join('\n') +
'define("templates", ["nunjucks", "helpers"], function(nunjucks) {\n' +
' nunjucks.env = new nunjucks.Environment([], {autoescape: true});\n' +
' nunjucks.env.registerPrecompiled(templates);\n' +
' nunjucks.templates = templates;\n' +
' console.log("Templates loaded");\n' +
' return nunjucks;\n' +
'});\n' +
'})();'
);
});
}
module.exports.build = build;
module.exports.js = concatJS;
module.exports.stylus = compileStylus;
module.exports.css = concatCSS;
module.exports.html = compileHTML;

156
lib/commonplace.js
Просмотреть файл

@ -176,7 +176,9 @@ function update() {
}
function clean() {
_check_version(info.src_dir(), undefined, undefined, function() {
var src_dir = info.src_dir();
check_version(src_dir, undefined, undefined, function() {
console.error('No Commonplace installation found.');
process.exit(1);
});
@ -188,19 +190,20 @@ function clean() {
'src/media/js/include.js',
'src/locales/'
];
targets.forEach(function(path) {
fs.stat(path, function(err, data) {
targets.forEach(function(filePath) {
filePath = path.resolve(src_dir, filePath);
fs.stat(filePath, function(err, data) {
if (err) return;
if (data && data.isDirectory()) {
utils.rmdirRecursive(path);
utils.rmdirRecursive(filePath);
} else {
utils.removeFile(path);
utils.removeFile(filePath);
}
});
});
var css_dir = 'src/media/css/';
var css_dir = path.resolve(src_dir, 'src/media/css/');
fs.exists(css_dir, function(exists) {
if (!exists) {
console.warn('CSS directory does not exist.');
@ -316,52 +319,124 @@ function extract_l10n() {
});
}
function build_includes(raw) {
function compile(options) {
var src_dir = info.src_dir();
_check_version(src_dir, undefined, undefined, function() {
if (!options || !options.silent) {
check_version(src_dir, undefined, function() {
console.warn('Found different commonplace version.');
console.warn('Generated includes may not work as expected.');
}, function() {
console.error('No Commonplace installation found.');
process.exit(1);
});
}
var build = require('./build.js');
var todo = (options && options.only) || ['stylus', 'nunjucks'];
var remaining = todo.length;
todo.forEach(function(v) {
switch (v) {
case 'stylus':
utils.globEach(src_dir, '.styl', function(file) {
// For every stylus file, increase the pending operation count.
// That way, the callback won't fire until everything is done.
remaining++;
build.stylus(file, function(err, css) {
if (err) {
console.error(err);
return;
}
fs.writeFile(file + '.css', css, function(err) {
if (err) {
console.error('Error writing CSS file: ' + file + '.css');
console.error(err);
return;
}
remaining--;
if (!remaining && options && options.callback) options.callback();
});
});
}, function() {
remaining--;
if (!remaining && options && options.callback) options.callback();
});
break;
case 'nunjucks':
build.html(src_dir, function(data) {
fs.writeFile(path.resolve(src_dir, 'templates.js'), data, function(err) {
if (err) {
console.error('Error writing templates file.', err);
return;
}
remaining--;
if (!remaining && options && options.callback) options.callback();
});
});
}
});
}
function build_includes() {
var raw = utils.opts().raw;
var src_dir = info.src_dir();
check_version(src_dir, undefined, function() {
console.warn('Found different commonplace version.');
console.warn('Generated includes may not work as expected.');
}, function() {
console.error('No Commonplace installation found.');
process.exit(1);
});
utils.copyFile(
__dirname + '/amd/amd.js',
src_dir + '/media/js/include.js',
function(err) {
var build = require('./build.js');
fs.readFile(
path.resolve(__dirname, 'assets/amd.js'),
function(err, amd_data) {
if (err) {
console.warn('Error copying amd.js');
console.warn('Error reading `amd.js`.', err);
return;
}
require('./build.js').build(src_dir);
// For raw includes mode. Does not minify.
if (raw) {return;}
var cleanCSS = require('clean-css');
var cssSource = src_dir + '/media/include.css';
console.log('Minifying CSS and writing to `media/css/include.css`');
fs.readFile(cssSource, function(err, data) {
if (err) {
console.warn('Error reading file: ' + cssSource);
}
var out = cleanCSS.process(data.toString());
fs.writeFile(src_dir + '/media/css/include.css', out, function(err) {
if (err) {
console.warn('Error writing `include.css` to disk.');
build.js(src_dir, function(data) {
// You need a function here, trust me. replace() is dumb and freaks out on dollar signs.
data = amd_data.toString().replace(/'replace me'/, function() {return data;});
if (!raw) {
try {
data = require('uglify-js').minify(
data, {screw_ie8: true, fromString: true}
).code;
} catch(e) {
console.error('Error during minification.', e);
}
});
});
console.log('Minifying JS and writing to `media/js/include.js`');
var result = require('uglify-js').minify(src_dir + '/media/include.js', {
screw_ie8: true
});
fs.writeFile(src_dir + '/media/js/include.js', result.code, function(err) {
if (err) {
console.warn('Error writing `include.js` to disk.');
}
var include_js = path.resolve(src_dir, 'media/js/include.js');
fs.writeFile(include_js, data, function(err) {
if (err) {
console.warn('Error writing `include.js` to disk.');
return;
}
console.log('Created ' + include_js);
});
});
}
);
build.css(src_dir, function(data) {
if (!raw) {
data = require('clean-css').process(data);
}
var include_css = path.resolve(src_dir, 'media/css/include.css');
fs.writeFile(include_css, data, function(err) {
if (err) {
console.warn('Error writing `include.css` to disk.');
return;
}
console.log('Created ' + include_css);
});
});
}
function wrap(func) {
@ -378,4 +453,5 @@ module.exports.update = wrap(update);
module.exports.clean = wrap(clean);
module.exports.generate_langpacks = wrap(generate_langpacks);
module.exports.extract_l10n = wrap(extract_l10n);
module.exports.compile = wrap(compile);
module.exports.build_includes = wrap(build_includes);

64
lib/compile.js
Просмотреть файл

@ -1,64 +0,0 @@
#!/usr/bin/env node
var fs = require('fs');
var path = require('path');
// nunjucks
var compiler = require('nunjucks').compiler;
var parser = require('nunjucks').parser;
var utils = require('./utils');
var srcdir = require('./info').src_dir();
function process(folder, output_file, callback) {
var extensions = require('./deferparser').extensions || [];
utils.glob(folder, '.html', function(err, templates) {
var template_strings = (
'(function() {' +
'var templates = {};\n'
);
for(var i=0; i<templates.length; i++) {
var name = templates[i].replace(path.join(folder, '/'), '');
template_strings += 'templates["' + name + '"] = (function() {';
var doCompile = function() {
var src = fs.readFileSync(templates[i], 'utf-8');
var cinst = new compiler.Compiler(extensions);
cinst.compile(parser.parse(src, extensions));
template_strings += cinst.getCode();
};
try {
doCompile();
} catch(e) {
template_strings += [
'return {root: function() {',
'throw new Error("' + name + ' failed to compile. Check the damper for details.");',
'}}'
].join('\n');
console.error(e);
}
template_strings += '})();\n';
}
template_strings += (
'define("templates", ["nunjucks", "helpers"], function(nunjucks) {\n' +
' nunjucks.env = new nunjucks.Environment([], {autoescape: true});\n' +
' nunjucks.env.registerPrecompiled(templates);\n' +
' nunjucks.templates = templates;\n' +
' console.log("Templates loaded");\n' +
' return nunjucks;\n' +
'});\n' +
'})();'
);
fs.writeFile(output_file, template_strings, callback);
});
}
module.exports.process = process;

4
lib/info.js
Просмотреть файл

@ -11,7 +11,7 @@ module.exports.src_dir = function() {
}
};
module.exports.version = function() {
var version_ = module.exports.version = function() {
var package_json = path.resolve(__dirname, '../package.json');
return JSON.parse(fs.readFileSync(package_json)).version;
};
@ -20,7 +20,7 @@ module.exports.check_version = function(src_dir, same, different, neither) {
var existing_manifest = path.resolve(src_dir, '.commonplace');
if (fs.existsSync(existing_manifest)) {
var version = JSON.parse(fs.readFileSync(existing_manifest)).version;
var current_version = info.version();
var current_version = version_();
if (version !== current_version && different) {
different(version, current_version);
} else if (version === current_version && same) {