From 3efa8fbbc5eda7de2c76968aff9ac16a21463ff2 Mon Sep 17 00:00:00 2001 From: Matt Basta Date: Tue, 13 Aug 2013 17:32:51 -0700 Subject: [PATCH] Rewrite build tools to be more async and centralized --- .gitignore | 2 +- bin/commonplace | 8 +- bin/damper | 194 +++++++++++++++---------------------- lib/{amd => assets}/amd.js | 0 lib/build.js | 191 ++++++++++++++++++++++++++---------- lib/commonplace.js | 156 +++++++++++++++++++++-------- lib/compile.js | 64 ------------ lib/info.js | 4 +- 8 files changed, 341 insertions(+), 278 deletions(-) rename lib/{amd => assets}/amd.js (100%) delete mode 100644 lib/compile.js diff --git a/.gitignore b/.gitignore index 85b467f..a9b60bd 100644 --- a/.gitignore +++ b/.gitignore @@ -23,9 +23,9 @@ build dist src/locales/*.js src/media/css/include.css +src/media/js/include.js src/media/include.css src/media/include.js -src/media/js/include.js src/templates.js node_modules npm-debug.log diff --git a/bin/commonplace b/bin/commonplace index 7d85046..c95704e 100755 --- a/bin/commonplace +++ b/bin/commonplace @@ -10,8 +10,8 @@ function help() { ' update This command will update commonplace modules.', ' extract_strings This command will extract strings into a `.pot` file.', ' langpacks This command will generate langpacks out of `.po` files.', - ' clean Run this to clean up static files, templates and langpacks.', - ' raw_includes Combine all the CSS and JS assets', + ' clean Clean up static files, templates, and langpacks.', + ' compile Generate CSS from stylus and JS from templates.', ' includes Combine and minify all the CSS and JS assets.' ].join('\n')); } @@ -43,8 +43,8 @@ switch (argv[0]) { case 'langpacks': commonplace.generate_langpacks(); break; - case 'raw_includes': - commonplace.build_includes(true); + case 'compile': + commonplace.compile(); break; case 'includes': commonplace.build_includes(); diff --git a/bin/damper b/bin/damper index cf6162c..41b37ad 100755 --- a/bin/damper +++ b/bin/damper @@ -2,13 +2,13 @@ var fs = require('fs'); var http = require('http'); +var path = require('path'); + var stylus = require('stylus'); var utils = require('../lib/utils.js'); var info = require('../lib/info.js'); -var opts = utils.opts; -var glob = utils.glob; var src_dir = require('../lib/info.js').src_dir(); process.title = 'damper'; @@ -35,150 +35,110 @@ info.check_version( } ); -var opts = opts(process.argv.slice(2), - {host: '0.0.0.0', port: '8675', compile: false}); +var opts = utils.opts(process.argv.slice(2), + {host: '0.0.0.0', port: '8675'}); -if (!opts.compile) { - // Here's the local server. - http.createServer(function(request, response) { +// Here's the local server. +http.createServer(function(request, response) { - var now = new Date(); + var now = new Date(); - console.log( - '[' + now.getHours() + ':' + now.getMinutes() + ':' + now.getSeconds() + '] ' + - request.url); + console.log( + '[' + now.getHours() + ':' + now.getMinutes() + ':' + now.getSeconds() + '] ' + + request.url); - function writeIndex() { - fs.readFile(src_dir + '/index.html', function(error, content) { - // We'll assume that you don't delete index.html. - response.writeHead(200, {'Content-Type': 'text/html'}); - response.end(content, 'utf-8'); - }); - } - - if (request.url == '/') { - return writeIndex(); - } - - var qindex; - if ((qindex = request.url.indexOf('?')) !== -1) { - request.url = request.url.substr(0, qindex); - } - - var filePath = src_dir + request.url; - fs.exists(filePath, function(exists) { - if (exists && !fs.statSync(filePath).isDirectory()) { - fs.readFile(filePath, function(error, content) { - if (error) { - response.writeHead(500); - response.end(); - console.error(error); - } else { - var dot = request.url.lastIndexOf('.'); - if (dot > -1) { - var extension = request.url.substr(dot + 1); - response.writeHead(200, {'Content-Type': mimes[extension]}); - } - - response.end(content, 'utf-8'); - } - }); - } else { - writeIndex(); - } + function writeIndex() { + fs.readFile(path.resolve(src_dir, 'index.html'), function(error, content) { + // We'll assume that you don't delete index.html. + response.writeHead(200, {'Content-Type': 'text/html'}); + response.end(content, 'utf-8'); }); + } - }).listen(opts.port, opts.host); + if (request.url == '/') { + return writeIndex(); + } - console.log('Server running at http://' + opts.host + ':' + opts.port); -} else { - console.log('Starting compilation...'); -} + var qindex; + if ((qindex = request.url.indexOf('?')) !== -1) { + request.url = request.url.substr(0, qindex); + } + + var url_path = request.url; + if (url_path[0] === '/') { + url_path = url_path.substr(1); + } + var filePath = path.resolve(src_dir, url_path); + fs.exists(filePath, function(exists) { + if (!exists || fs.statSync(filePath).isDirectory()) { + writeIndex(); + return; + } + + fs.readFile(filePath, function(err, content) { + if (err) { + response.writeHead(500); + response.end(); + console.error(err); + return; + } + var dot = request.url.lastIndexOf('.'); + if (dot > -1) { + var extension = request.url.substr(dot + 1); + response.writeHead(200, {'Content-Type': mimes[extension]}); + } + + response.end(content, 'utf-8'); + }); + }); + +}).listen(opts.port, opts.host); + +console.log('Server running at http://' + opts.host + ':' + opts.port); -var watched_filepaths = []; function runCommand(command, filepath) { switch (command) { case 'stylus': - filepath = filepath || opts.path; - - var filepathDir = filepath.split('/').slice(0, -1).join('/'); - fs.readFile(filepath, function (err, data) { - data = data.toString(); - if (err) { - console.error(filepath + ' not found!'); - return; - } - stylus(data) - .set('filename', filepath + '.css') - .set('include css', true) - .include(filepathDir) - .render(function(err, css) { - fs.writeFileSync(filepath + '.css', css); - if (err) console.error(err); - }); + require('../lib/build').stylus(filepath, function(err, css) { + fs.writeFileSync(filepath + '.css', css); + if (err) console.error(err); }); break; case 'nunjucks': console.log('Recompiling templates...'); - require('../lib/compile').process( - src_dir + '/templates', - src_dir + '/templates.js' - ); + require('../lib/commonplace').compile({silent: true, only: ['nunjucks']}); break; } } function watch(globpath, ext, command) { - var cb = function(err, filepaths) { - filepaths.forEach(function(filepath) { - watched_filepaths.push(filepath); - if (command == 'stylus') { - fs.exists(filepath, function(exists) { - if (exists) { - runCommand(command, filepath); - } - }); - } + fs.exists(globpath, function(exists) { + if (!exists) { + console.warn('Skipping absent path: ' + globpath); + return; + } - // If we're compiling, we don't want to watch the files. - if (opts.compile) { - return; + var count = 0; + utils.globEach(globpath, ext, function(filepath) { + count++; + if (command == 'stylus') { + runCommand(command, filepath); } fs.watchFile(filepath, {interval: 250}, function(curr, prev) { - if (curr.mtime.valueOf() != prev.mtime.valueOf() || - curr.ctime.valueOf() != prev.ctime.valueOf()) { + if (curr.mtime.valueOf() !== prev.mtime.valueOf() || + curr.ctime.valueOf() !== prev.ctime.valueOf()) { console.warn('> ' + filepath + ' changed.'); runCommand(command, filepath); } }); - + }, function() { + console.log('Watching ' + count + ' `' + ext + '` files.'); }); - if (filepaths.length > 1 && !opts.compile) { - console.log('Watching ' + filepaths.length + ' ' + ext + ' files.'); - } - }; - if (!fs.existsSync(globpath)) { - console.warn('Skipping non-existing path: ' + globpath); - return; - } - if (globpath.substr(1).indexOf('.') > -1) { - cb(null, [globpath]); - } else { - glob(globpath, ext, cb); - } -} - -if (opts.compile && opts.compile !== true) { - runCommand(opts.compile); -} else { - runCommand('nunjucks'); - - watch(src_dir + '/media/css', 'styl', 'stylus'); - watch(src_dir + '/templates', 'html', 'nunjucks'); - - // When the builder is updated, recompile the templates. - watch(src_dir + '/media/js/builder.js', null, 'nunjucks'); + }); } +runCommand('nunjucks'); +watch(path.resolve(src_dir, 'media/css'), 'styl', 'stylus'); +watch(path.resolve(src_dir, 'templates'), 'html', 'nunjucks'); diff --git a/lib/amd/amd.js b/lib/assets/amd.js similarity index 100% rename from lib/amd/amd.js rename to lib/assets/amd.js diff --git a/lib/build.js b/lib/build.js index 617e5a7..c437aab 100644 --- a/lib/build.js +++ b/lib/build.js @@ -1,5 +1,6 @@ var fs = require('fs'); var path = require('path'); + var utils = require('./utils.js'); var blacklist = [ @@ -9,72 +10,162 @@ var blacklist = [ 'settings_travis.js', 'splash.styl.css', + 'templates.js', // Generated dynamically. 'include.js', 'include.css', ]; -function concatJS(assetsPath, includes, output_file) { +function concatJS(src_dir, callback) { var output = ''; - includes.forEach(function(include) { - include = assetsPath + '/' + include; - if (include.substr(-3) == '.js') { - output += fs.readFileSync(include).toString(); - } else { - utils.globSync(include, '.js', function(err, results) { - results.forEach(function(file) { - if (blacklist.indexOf(path.basename(file)) === -1) { - output += fs.readFileSync(file).toString() + '\n'; - } - }); - }); - } + // Render the HTML for the templates. + compileHTML(src_dir, function(html_data) { + output += html_data; + + // Now include the rest of the JS. + utils.globEach( + path.resolve(src_dir, 'media/js'), + '.js', + function(file) { + if (blacklist.indexOf(path.basename(file)) !== -1) return; + output += fs.readFileSync(file) + '\n'; + }, function() { + callback(output); + } + ); }); - var include = fs.readFileSync(assetsPath + '/' + output_file).toString(); - output = include.replace(/'replace me'/, function(){ - return output; - }); - fs.writeFileSync(assetsPath + '/' + output_file, output); } -function concatCSS(assetsPath, includes, output_file) { - var output = ''; - var match; - var css_pattern = /href="(\/media\/css\/.+\.styl\.css)"/g; +function compileStylus(source_path, callback) { + var stylus = require('stylus'); + fs.readFile(source_path, function(err, data) { + if (err) { + console.error('Could not read stylus file: ' + source_path); + callback(err); + return; + } - includes.forEach(function(include) { - var include_path = [assetsPath, '/', include].join(''); - var include_data = fs.readFileSync(include_path).toString(); - if (include.substr(-5) === '.html') { - while (match = css_pattern.exec(include_data)) { - if (blacklist.indexOf(path.basename(match[1])) === -1) { - output += fs.readFileSync(assetsPath + match[1]).toString(); + stylus(data.toString()) + .set('filename', source_path + '.css') + .set('include css', true) + .include(path.dirname(source_path)) + .render(function(err, css) { + if (err) { + console.error('Error compiling stylus file: ' + source_path); + callback(err); + return; } + callback(null, css); + }); + }); +} + +function concatCSS(src_dir, callback) { + var output = ''; + var css_pattern = /href="(\/media\/css\/.+\.styl\.css)"/g; + var url_pattern = /url\(([^)]+)\)/g; + + function fix_urls(data) { + return data.replace(url_pattern, function(match, url, offset, string) { + url = url.replace(/"|'/g, ''); + if (url.search(/(https?|data):|\/\//) === 0) { + return ['url(', url, ')'].join(''); + } + + var timestamp = new Date().getTime(); + if (url.indexOf('#') !== -1) { + var split = url.split('#'); + return ['url(', split[0], '?', timestamp, '#', split[1], ')'].join(''); + } else { + return ['url(', url, '?', timestamp, ')'].join(''); + } + }); + } + + var index_html = fs.readFile(path.resolve(src_dir, 'index.html'), function(err, data) { + if (err) { + console.error('Could not read `index.html`.', err); + return; + } + var match; + var files = []; + data = data.toString(); + while (match = css_pattern.exec(data)) { + if (blacklist.indexOf(path.basename(match[1])) === -1) { + files.push(path.resolve(src_dir + match[1])); } } - }); - var url_pattern = /url\(([^)]+)\)/g; - output = output.replace(url_pattern, function(match, url, offset, string) { - url = url.replace(/"|'/g, ''); - if (url.search(/(https?|data):|\/\//) === 0) { - return ['url(', url, ')'].join(''); - } + var remaining = files.length; + files.forEach(function(v) { + v = v.replace('.styl.css', '.styl'); + compileStylus(v, function(err, data) { + remaining--; + if (err) { + console.warn(err); - var timestamp = new Date().getTime(); - if (url.indexOf('#') !== -1) { - var split = url.split('#'); - return ['url(', split[0], '?', timestamp, '#', split[1], ')'].join(''); - } else { - return ['url(', url, '?', timestamp, ')'].join(''); - } + if (!remaining) callback(output); + return; + } + output += fix_urls(data + '\n'); + if (!remaining) callback(output); + }); + }); }); - fs.writeFileSync(assetsPath + '/' + output_file, output); } -function build(assetsPath) { - console.log('Building assets...'); - concatJS(assetsPath, ['media/js', 'templates.js'], 'media/include.js'); - concatCSS(assetsPath, ['index.html'], 'media/include.css'); +function compileHTML(src_dir, callback) { + var compiler = require('nunjucks').compiler; + var parser = require('nunjucks').parser; + + var extensions = require('./deferparser').extensions || []; + + var template_dir = path.resolve(src_dir, 'templates'); + if (template_dir.substr(-1) !== '/') { + template_dir += '/'; + } + + var template_data = []; + utils.globEach(template_dir, '.html', function(template) { + var name = template.replace(template_dir, ''); + var output = 'templates["' + name + '"] = (function() {'; + try { + var src = fs.readFileSync(template, 'utf-8'); + var cinst = new compiler.Compiler(extensions); + // Parse + var parsed = parser.parse(src, extensions); + // Compile + cinst.compile(parsed); + // Output + output += cinst.getCode(); + } catch(e) { + output += [ + 'return {root: function() {', + 'throw new Error("' + name + ' failed to compile. Check the damper for details.");', + '}}' + ].join('\n'); + + console.error(e); + } + + template_data.push(output + '})();\n'); + }, function() { + callback( + '(function() {' + + 'var templates = {};\n' + + template_data.join('\n') + + 'define("templates", ["nunjucks", "helpers"], function(nunjucks) {\n' + + ' nunjucks.env = new nunjucks.Environment([], {autoescape: true});\n' + + ' nunjucks.env.registerPrecompiled(templates);\n' + + ' nunjucks.templates = templates;\n' + + ' console.log("Templates loaded");\n' + + ' return nunjucks;\n' + + '});\n' + + '})();' + ); + }); } -module.exports.build = build; +module.exports.js = concatJS; +module.exports.stylus = compileStylus; +module.exports.css = concatCSS; +module.exports.html = compileHTML; diff --git a/lib/commonplace.js b/lib/commonplace.js index ae7d3c5..98c4a57 100644 --- a/lib/commonplace.js +++ b/lib/commonplace.js @@ -176,7 +176,9 @@ function update() { } function clean() { - _check_version(info.src_dir(), undefined, undefined, function() { + var src_dir = info.src_dir(); + + check_version(src_dir, undefined, undefined, function() { console.error('No Commonplace installation found.'); process.exit(1); }); @@ -188,19 +190,20 @@ function clean() { 'src/media/js/include.js', 'src/locales/' ]; - targets.forEach(function(path) { - fs.stat(path, function(err, data) { + targets.forEach(function(filePath) { + filePath = path.resolve(src_dir, filePath); + fs.stat(filePath, function(err, data) { if (err) return; if (data && data.isDirectory()) { - utils.rmdirRecursive(path); + utils.rmdirRecursive(filePath); } else { - utils.removeFile(path); + utils.removeFile(filePath); } }); }); - var css_dir = 'src/media/css/'; + var css_dir = path.resolve(src_dir, 'src/media/css/'); fs.exists(css_dir, function(exists) { if (!exists) { console.warn('CSS directory does not exist.'); @@ -316,52 +319,124 @@ function extract_l10n() { }); } -function build_includes(raw) { +function compile(options) { var src_dir = info.src_dir(); - _check_version(src_dir, undefined, undefined, function() { + + if (!options || !options.silent) { + check_version(src_dir, undefined, function() { + console.warn('Found different commonplace version.'); + console.warn('Generated includes may not work as expected.'); + }, function() { + console.error('No Commonplace installation found.'); + process.exit(1); + }); + } + + var build = require('./build.js'); + + var todo = (options && options.only) || ['stylus', 'nunjucks']; + var remaining = todo.length; + todo.forEach(function(v) { + switch (v) { + case 'stylus': + utils.globEach(src_dir, '.styl', function(file) { + // For every stylus file, increase the pending operation count. + // That way, the callback won't fire until everything is done. + remaining++; + build.stylus(file, function(err, css) { + if (err) { + console.error(err); + return; + } + fs.writeFile(file + '.css', css, function(err) { + if (err) { + console.error('Error writing CSS file: ' + file + '.css'); + console.error(err); + return; + } + + remaining--; + if (!remaining && options && options.callback) options.callback(); + }); + }); + }, function() { + remaining--; + if (!remaining && options && options.callback) options.callback(); + }); + break; + case 'nunjucks': + build.html(src_dir, function(data) { + fs.writeFile(path.resolve(src_dir, 'templates.js'), data, function(err) { + if (err) { + console.error('Error writing templates file.', err); + return; + } + remaining--; + if (!remaining && options && options.callback) options.callback(); + }); + }); + } + }); + +} + +function build_includes() { + var raw = utils.opts().raw; + + var src_dir = info.src_dir(); + check_version(src_dir, undefined, function() { + console.warn('Found different commonplace version.'); + console.warn('Generated includes may not work as expected.'); + }, function() { console.error('No Commonplace installation found.'); process.exit(1); }); - utils.copyFile( - __dirname + '/amd/amd.js', - src_dir + '/media/js/include.js', - function(err) { + var build = require('./build.js'); + fs.readFile( + path.resolve(__dirname, 'assets/amd.js'), + function(err, amd_data) { if (err) { - console.warn('Error copying amd.js'); + console.warn('Error reading `amd.js`.', err); + return; } - require('./build.js').build(src_dir); - - // For raw includes mode. Does not minify. - if (raw) {return;} - - var cleanCSS = require('clean-css'); - var cssSource = src_dir + '/media/include.css'; - - console.log('Minifying CSS and writing to `media/css/include.css`'); - fs.readFile(cssSource, function(err, data) { - if (err) { - console.warn('Error reading file: ' + cssSource); - } - var out = cleanCSS.process(data.toString()); - fs.writeFile(src_dir + '/media/css/include.css', out, function(err) { - if (err) { - console.warn('Error writing `include.css` to disk.'); + build.js(src_dir, function(data) { + // You need a function here, trust me. replace() is dumb and freaks out on dollar signs. + data = amd_data.toString().replace(/'replace me'/, function() {return data;}); + if (!raw) { + try { + data = require('uglify-js').minify( + data, {screw_ie8: true, fromString: true} + ).code; + } catch(e) { + console.error('Error during minification.', e); } - }); - }); - - console.log('Minifying JS and writing to `media/js/include.js`'); - var result = require('uglify-js').minify(src_dir + '/media/include.js', { - screw_ie8: true - }); - fs.writeFile(src_dir + '/media/js/include.js', result.code, function(err) { - if (err) { - console.warn('Error writing `include.js` to disk.'); } + var include_js = path.resolve(src_dir, 'media/js/include.js'); + fs.writeFile(include_js, data, function(err) { + if (err) { + console.warn('Error writing `include.js` to disk.'); + return; + } + console.log('Created ' + include_js); + }); }); } ); + + build.css(src_dir, function(data) { + if (!raw) { + data = require('clean-css').process(data); + } + var include_css = path.resolve(src_dir, 'media/css/include.css'); + fs.writeFile(include_css, data, function(err) { + if (err) { + console.warn('Error writing `include.css` to disk.'); + return; + } + console.log('Created ' + include_css); + }); + }); } function wrap(func) { @@ -378,4 +453,5 @@ module.exports.update = wrap(update); module.exports.clean = wrap(clean); module.exports.generate_langpacks = wrap(generate_langpacks); module.exports.extract_l10n = wrap(extract_l10n); +module.exports.compile = wrap(compile); module.exports.build_includes = wrap(build_includes); diff --git a/lib/compile.js b/lib/compile.js deleted file mode 100644 index ffedb20..0000000 --- a/lib/compile.js +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env node -var fs = require('fs'); -var path = require('path'); - -// nunjucks -var compiler = require('nunjucks').compiler; -var parser = require('nunjucks').parser; - -var utils = require('./utils'); -var srcdir = require('./info').src_dir(); - -function process(folder, output_file, callback) { - var extensions = require('./deferparser').extensions || []; - - utils.glob(folder, '.html', function(err, templates) { - var template_strings = ( - '(function() {' + - 'var templates = {};\n' - ); - - for(var i=0; i