Rewrite build tools to be more async and centralized

.gitignore

@@ -23,9 +23,9 @@ build
 dist
 src/locales/*.js
 src/media/css/include.css
+src/media/js/include.js
 src/media/include.css
 src/media/include.js
-src/media/js/include.js
 src/templates.js
 node_modules
 npm-debug.log

bin/commonplace

@@ -10,8 +10,8 @@ function help() {
         '  update           This command will update commonplace modules.',
         '  extract_strings  This command will extract strings into a `.pot` file.',
         '  langpacks        This command will generate langpacks out of `.po` files.',
-        '  clean            Run this to clean up static files, templates and langpacks.',
+        '  clean            Clean up static files, templates, and langpacks.',
-        '  raw_includes     Combine all the CSS and JS assets',
+        '  compile          Generate CSS from stylus and JS from templates.',
         '  includes         Combine and minify all the CSS and JS assets.'
     ].join('\n'));
 }
@@ -43,8 +43,8 @@ switch (argv[0]) {
     case 'langpacks':
         commonplace.generate_langpacks();
         break;
-    case 'raw_includes':
+    case 'compile':
-        commonplace.build_includes(true);
+        commonplace.compile();
         break;
     case 'includes':
         commonplace.build_includes();

bin/damper

@@ -2,13 +2,13 @@
 
 var fs = require('fs');
 var http = require('http');
+var path = require('path');
+
 var stylus = require('stylus');
 
 var utils = require('../lib/utils.js');
 var info = require('../lib/info.js');
 
-var opts = utils.opts;
-var glob = utils.glob;
 var src_dir = require('../lib/info.js').src_dir();
 
 process.title = 'damper';
@@ -35,150 +35,110 @@ info.check_version(
     }
 );
 
-var opts = opts(process.argv.slice(2),
+var opts = utils.opts(process.argv.slice(2),
-                {host: '0.0.0.0', port: '8675', compile: false});
+                      {host: '0.0.0.0', port: '8675'});
 
-if (!opts.compile) {
+// Here's the local server.
-    // Here's the local server.
+http.createServer(function(request, response) {
-    http.createServer(function(request, response) {
 
-        var now = new Date();
+    var now = new Date();
 
-        console.log(
+    console.log(
-            '[' + now.getHours() + ':' + now.getMinutes() + ':' + now.getSeconds() + '] ' +
+        '[' + now.getHours() + ':' + now.getMinutes() + ':' + now.getSeconds() + '] ' +
-            request.url);
+        request.url);
 
-        function writeIndex() {
+    function writeIndex() {
-            fs.readFile(src_dir + '/index.html', function(error, content) {
+        fs.readFile(path.resolve(src_dir, 'index.html'), function(error, content) {
-                // We'll assume that you don't delete index.html.
+            // We'll assume that you don't delete index.html.
-                response.writeHead(200, {'Content-Type': 'text/html'});
+            response.writeHead(200, {'Content-Type': 'text/html'});
-                response.end(content, 'utf-8');
+            response.end(content, 'utf-8');
-            });
-        }
-
-        if (request.url == '/') {
-            return writeIndex();
-        }
-
-        var qindex;
-        if ((qindex = request.url.indexOf('?')) !== -1) {
-            request.url = request.url.substr(0, qindex);
-        }
-
-        var filePath = src_dir + request.url;
-        fs.exists(filePath, function(exists) {
-            if (exists && !fs.statSync(filePath).isDirectory()) {
-                fs.readFile(filePath, function(error, content) {
-                    if (error) {
-                        response.writeHead(500);
-                        response.end();
-                        console.error(error);
-                    } else {
-                        var dot = request.url.lastIndexOf('.');
-                        if (dot > -1) {
-                            var extension = request.url.substr(dot + 1);
-                            response.writeHead(200, {'Content-Type': mimes[extension]});
-                        }
-
-                        response.end(content, 'utf-8');
-                    }
-                });
-            } else {
-                writeIndex();
-            }
         });
+    }
 
-    }).listen(opts.port, opts.host);
+    if (request.url == '/') {
+        return writeIndex();
+    }
 
-    console.log('Server running at http://' + opts.host + ':' + opts.port);
+    var qindex;
-} else {
+    if ((qindex = request.url.indexOf('?')) !== -1) {
-    console.log('Starting compilation...');
+        request.url = request.url.substr(0, qindex);
-}
+    }
+
+    var url_path = request.url;
+    if (url_path[0] === '/') {
+        url_path = url_path.substr(1);
+    }
+    var filePath = path.resolve(src_dir, url_path);
+    fs.exists(filePath, function(exists) {
+        if (!exists || fs.statSync(filePath).isDirectory()) {
+            writeIndex();
+            return;
+        }
+
+        fs.readFile(filePath, function(err, content) {
+            if (err) {
+                response.writeHead(500);
+                response.end();
+                console.error(err);
+                return;
+            }
+            var dot = request.url.lastIndexOf('.');
+            if (dot > -1) {
+                var extension = request.url.substr(dot + 1);
+                response.writeHead(200, {'Content-Type': mimes[extension]});
+            }
+
+            response.end(content, 'utf-8');
+        });
+    });
+
+}).listen(opts.port, opts.host);
+
+console.log('Server running at http://' + opts.host + ':' + opts.port);
 
-var watched_filepaths = [];
 
 function runCommand(command, filepath) {
     switch (command) {
         case 'stylus':
-            filepath = filepath || opts.path;
+            require('../lib/build').stylus(filepath, function(err, css) {
-
+                fs.writeFileSync(filepath + '.css', css);
-            var filepathDir = filepath.split('/').slice(0, -1).join('/');
+                if (err) console.error(err);
-            fs.readFile(filepath, function (err, data) {
-                data = data.toString();
-                if (err) {
-                    console.error(filepath + ' not found!');
-                    return;
-                }
-                stylus(data)
-                    .set('filename', filepath + '.css')
-                    .set('include css', true)
-                    .include(filepathDir)
-                    .render(function(err, css) {
-                        fs.writeFileSync(filepath + '.css', css);
-                        if (err) console.error(err);
-                    });
             });
             break;
         case 'nunjucks':
             console.log('Recompiling templates...');
-            require('../lib/compile').process(
+            require('../lib/commonplace').compile({silent: true, only: ['nunjucks']});
-                src_dir + '/templates',
-                src_dir + '/templates.js'
-            );
             break;
     }
 }
 
 function watch(globpath, ext, command) {
-    var cb = function(err, filepaths) {
+    fs.exists(globpath, function(exists) {
-        filepaths.forEach(function(filepath) {
+        if (!exists) {
-            watched_filepaths.push(filepath);
+            console.warn('Skipping absent path: ' + globpath);
-            if (command == 'stylus') {
+            return;
-                fs.exists(filepath, function(exists) {
+        }
-                    if (exists) {
-                        runCommand(command, filepath);
-                    }
-                });
-            }
 
-            // If we're compiling, we don't want to watch the files.
+        var count = 0;
-            if (opts.compile) {
+        utils.globEach(globpath, ext, function(filepath) {
-                return;
+            count++;
+            if (command == 'stylus') {
+                runCommand(command, filepath);
             }
 
             fs.watchFile(filepath, {interval: 250}, function(curr, prev) {
-                if (curr.mtime.valueOf() != prev.mtime.valueOf() ||
+                if (curr.mtime.valueOf() !== prev.mtime.valueOf() ||
-                    curr.ctime.valueOf() != prev.ctime.valueOf()) {
+                    curr.ctime.valueOf() !== prev.ctime.valueOf()) {
                     console.warn('> ' + filepath + ' changed.');
                     runCommand(command, filepath);
                 }
             });
-
+        }, function() {
+            console.log('Watching ' + count + ' `' + ext + '` files.');
         });
-        if (filepaths.length > 1 && !opts.compile) {
+    });
-            console.log('Watching ' + filepaths.length + ' ' + ext + ' files.');
-        }
-    };
-    if (!fs.existsSync(globpath)) {
-        console.warn('Skipping non-existing path: ' + globpath);
-        return;
-    }
-    if (globpath.substr(1).indexOf('.') > -1) {
-        cb(null, [globpath]);
-    } else {
-        glob(globpath, ext, cb);
-    }
-}
-
-if (opts.compile && opts.compile !== true) {
-    runCommand(opts.compile);
-} else {
-    runCommand('nunjucks');
-
-    watch(src_dir + '/media/css', 'styl', 'stylus');
-    watch(src_dir + '/templates', 'html', 'nunjucks');
-
-    // When the builder is updated, recompile the templates.
-    watch(src_dir + '/media/js/builder.js', null, 'nunjucks');
 }
 
+runCommand('nunjucks');
+watch(path.resolve(src_dir, 'media/css'), 'styl', 'stylus');
+watch(path.resolve(src_dir, 'templates'), 'html', 'nunjucks');

lib/build.js

@@ -1,5 +1,6 @@
 var fs = require('fs');
 var path = require('path');
+
 var utils = require('./utils.js');
 
 var blacklist = [
@@ -9,72 +10,162 @@ var blacklist = [
     'settings_travis.js',
 
     'splash.styl.css',
+    'templates.js',  // Generated dynamically.
     'include.js',
     'include.css',
 ];
 
-function concatJS(assetsPath, includes, output_file) {
+function concatJS(src_dir, callback) {
     var output = '';
-    includes.forEach(function(include) {
+    // Render the HTML for the templates.
-        include = assetsPath + '/' + include;
+    compileHTML(src_dir, function(html_data) {
-        if (include.substr(-3) == '.js') {
+        output += html_data;
-            output += fs.readFileSync(include).toString();
+
-        } else {
+        // Now include the rest of the JS.
-            utils.globSync(include, '.js', function(err, results) {
+        utils.globEach(
-                results.forEach(function(file) {
+            path.resolve(src_dir, 'media/js'),
-                    if (blacklist.indexOf(path.basename(file)) === -1) {
+            '.js',
-                        output += fs.readFileSync(file).toString() + '\n';
+            function(file) {
-                    }
+                if (blacklist.indexOf(path.basename(file)) !== -1) return;
-                });
+                output += fs.readFileSync(file) + '\n';
-            });
+            }, function() {
-        }
+                callback(output);
+            }
+        );
     });
-    var include = fs.readFileSync(assetsPath + '/' + output_file).toString();
-    output = include.replace(/'replace me'/, function(){
-        return output;
-    });
-    fs.writeFileSync(assetsPath + '/' + output_file, output);
 }
 
-function concatCSS(assetsPath, includes, output_file) {
+function compileStylus(source_path, callback) {
-    var output = '';
+    var stylus = require('stylus');
-    var match;
+    fs.readFile(source_path, function(err, data) {
-    var css_pattern = /href="(\/media\/css\/.+\.styl\.css)"/g;
+        if (err) {
+            console.error('Could not read stylus file: ' + source_path);
+            callback(err);
+            return;
+        }
 
-    includes.forEach(function(include) {
+        stylus(data.toString())
-        var include_path = [assetsPath, '/', include].join('');
+            .set('filename', source_path + '.css')
-        var include_data = fs.readFileSync(include_path).toString();
+            .set('include css', true)
-        if (include.substr(-5) === '.html') {
+            .include(path.dirname(source_path))
-            while (match = css_pattern.exec(include_data)) {
+            .render(function(err, css) {
-                if (blacklist.indexOf(path.basename(match[1])) === -1) {
+                if (err) {
-                    output += fs.readFileSync(assetsPath + match[1]).toString();
+                    console.error('Error compiling stylus file: ' + source_path);
+                    callback(err);
+                    return;
                 }
+                callback(null, css);
+            });
+    });
+}
+
+function concatCSS(src_dir, callback) {
+    var output = '';
+    var css_pattern = /href="(\/media\/css\/.+\.styl\.css)"/g;
+    var url_pattern = /url\(([^)]+)\)/g;
+
+    function fix_urls(data) {
+        return data.replace(url_pattern, function(match, url, offset, string) {
+            url = url.replace(/"|'/g, '');
+            if (url.search(/(https?|data):|\/\//) === 0) {
+                return ['url(', url, ')'].join('');
+            }
+
+            var timestamp = new Date().getTime();
+            if (url.indexOf('#') !== -1) {
+                var split = url.split('#');
+                return ['url(', split[0], '?', timestamp, '#', split[1], ')'].join('');
+            } else {
+                return ['url(', url, '?', timestamp, ')'].join('');
+            }
+        });
+    }
+
+    var index_html = fs.readFile(path.resolve(src_dir, 'index.html'), function(err, data) {
+        if (err) {
+            console.error('Could not read `index.html`.', err);
+            return;
+        }
+        var match;
+        var files = [];
+        data = data.toString();
+        while (match = css_pattern.exec(data)) {
+            if (blacklist.indexOf(path.basename(match[1])) === -1) {
+                files.push(path.resolve(src_dir + match[1]));
             }
         }
-    });
 
-    var url_pattern = /url\(([^)]+)\)/g;
+        var remaining = files.length;
-    output = output.replace(url_pattern, function(match, url, offset, string) {
+        files.forEach(function(v) {
-        url = url.replace(/"|'/g, '');
+            v = v.replace('.styl.css', '.styl');
-        if (url.search(/(https?|data):|\/\//) === 0) {
+            compileStylus(v, function(err, data) {
-            return ['url(', url, ')'].join('');
+                remaining--;
-        }
+                if (err) {
+                    console.warn(err);
 
-        var timestamp = new Date().getTime();
+                    if (!remaining) callback(output);
-        if (url.indexOf('#') !== -1) {
+                    return;
-            var split = url.split('#');
+                }
-            return ['url(', split[0], '?', timestamp, '#', split[1], ')'].join('');
+                output += fix_urls(data + '\n');
-        } else {
+                if (!remaining) callback(output);
-            return ['url(', url, '?', timestamp, ')'].join('');
+            });
-        }
+        });
     });
-    fs.writeFileSync(assetsPath + '/' + output_file, output);
 }
 
-function build(assetsPath) {
+function compileHTML(src_dir, callback) {
-    console.log('Building assets...');
+    var compiler = require('nunjucks').compiler;
-    concatJS(assetsPath, ['media/js', 'templates.js'], 'media/include.js');
+    var parser = require('nunjucks').parser;
-    concatCSS(assetsPath, ['index.html'], 'media/include.css');
+
+    var extensions = require('./deferparser').extensions || [];
+
+    var template_dir = path.resolve(src_dir, 'templates');
+    if (template_dir.substr(-1) !== '/') {
+        template_dir += '/';
+    }
+
+    var template_data = [];
+    utils.globEach(template_dir, '.html', function(template) {
+        var name = template.replace(template_dir, '');
+        var output = 'templates["' + name + '"] = (function() {';
+        try {
+            var src = fs.readFileSync(template, 'utf-8');
+            var cinst = new compiler.Compiler(extensions);
+            // Parse
+            var parsed = parser.parse(src, extensions);
+            // Compile
+            cinst.compile(parsed);
+            // Output
+            output += cinst.getCode();
+        } catch(e) {
+            output += [
+                'return {root: function() {',
+                'throw new Error("' + name + ' failed to compile. Check the damper for details.");',
+                '}}'
+            ].join('\n');
+
+            console.error(e);
+        }
+
+        template_data.push(output + '})();\n');
+    }, function() {
+        callback(
+            '(function() {' +
+            'var templates = {};\n' +
+            template_data.join('\n') +
+            'define("templates", ["nunjucks", "helpers"], function(nunjucks) {\n' +
+            '    nunjucks.env = new nunjucks.Environment([], {autoescape: true});\n' +
+            '    nunjucks.env.registerPrecompiled(templates);\n' +
+            '    nunjucks.templates = templates;\n' +
+            '    console.log("Templates loaded");\n' +
+            '    return nunjucks;\n' +
+            '});\n' +
+            '})();'
+        );
+    });
 }
 
-module.exports.build = build;
+module.exports.js = concatJS;
+module.exports.stylus = compileStylus;
+module.exports.css = concatCSS;
+module.exports.html = compileHTML;

lib/commonplace.js

@@ -176,7 +176,9 @@ function update() {
 }
 
 function clean() {
-    _check_version(info.src_dir(), undefined, undefined, function() {
+    var src_dir = info.src_dir();
+
+    check_version(src_dir, undefined, undefined, function() {
         console.error('No Commonplace installation found.');
         process.exit(1);
     });
@@ -188,19 +190,20 @@ function clean() {
         'src/media/js/include.js',
         'src/locales/'
     ];
-    targets.forEach(function(path) {
+    targets.forEach(function(filePath) {
-        fs.stat(path, function(err, data) {
+        filePath = path.resolve(src_dir, filePath);
+        fs.stat(filePath, function(err, data) {
             if (err) return;
 
             if (data && data.isDirectory()) {
-                utils.rmdirRecursive(path);
+                utils.rmdirRecursive(filePath);
             } else {
-                utils.removeFile(path);
+                utils.removeFile(filePath);
             }
         });
     });
 
-    var css_dir = 'src/media/css/';
+    var css_dir = path.resolve(src_dir, 'src/media/css/');
     fs.exists(css_dir, function(exists) {
         if (!exists) {
             console.warn('CSS directory does not exist.');
@@ -316,52 +319,124 @@ function extract_l10n() {
     });
 }
 
-function build_includes(raw) {
+function compile(options) {
     var src_dir = info.src_dir();
-    _check_version(src_dir, undefined, undefined, function() {
+
+    if (!options || !options.silent) {
+        check_version(src_dir, undefined, function() {
+            console.warn('Found different commonplace version.');
+            console.warn('Generated includes may not work as expected.');
+        }, function() {
+            console.error('No Commonplace installation found.');
+            process.exit(1);
+        });
+    }
+
+    var build = require('./build.js');
+
+    var todo = (options && options.only) || ['stylus', 'nunjucks'];
+    var remaining = todo.length;
+    todo.forEach(function(v) {
+        switch (v) {
+            case 'stylus':
+                utils.globEach(src_dir, '.styl', function(file) {
+                    // For every stylus file, increase the pending operation count.
+                    // That way, the callback won't fire until everything is done.
+                    remaining++;
+                    build.stylus(file, function(err, css) {
+                        if (err) {
+                            console.error(err);
+                            return;
+                        }
+                        fs.writeFile(file + '.css', css, function(err) {
+                            if (err) {
+                                console.error('Error writing CSS file: ' + file + '.css');
+                                console.error(err);
+                                return;
+                            }
+
+                            remaining--;
+                            if (!remaining && options && options.callback) options.callback();
+                        });
+                    });
+                }, function() {
+                    remaining--;
+                    if (!remaining && options && options.callback) options.callback();
+                });
+                break;
+            case 'nunjucks':
+                build.html(src_dir, function(data) {
+                    fs.writeFile(path.resolve(src_dir, 'templates.js'), data, function(err) {
+                        if (err) {
+                            console.error('Error writing templates file.', err);
+                            return;
+                        }
+                        remaining--;
+                        if (!remaining && options && options.callback) options.callback();
+                    });
+                });
+        }
+    });
+
+}
+
+function build_includes() {
+    var raw = utils.opts().raw;
+
+    var src_dir = info.src_dir();
+    check_version(src_dir, undefined, function() {
+        console.warn('Found different commonplace version.');
+        console.warn('Generated includes may not work as expected.');
+    }, function() {
         console.error('No Commonplace installation found.');
         process.exit(1);
     });
 
-    utils.copyFile(
+    var build = require('./build.js');
-        __dirname + '/amd/amd.js',
+    fs.readFile(
-        src_dir + '/media/js/include.js',
+        path.resolve(__dirname, 'assets/amd.js'),
-        function(err) {
+        function(err, amd_data) {
             if (err) {
-                console.warn('Error copying amd.js');
+                console.warn('Error reading `amd.js`.', err);
+                return;
             }
-            require('./build.js').build(src_dir);
+            build.js(src_dir, function(data) {
-
+                // You need a function here, trust me. replace() is dumb and freaks out on dollar signs.
-            // For raw includes mode. Does not minify.
+                data = amd_data.toString().replace(/'replace me'/, function() {return data;});
-            if (raw) {return;}
+                if (!raw) {
-
+                    try {
-            var cleanCSS = require('clean-css');
+                        data = require('uglify-js').minify(
-            var cssSource = src_dir + '/media/include.css';
+                            data, {screw_ie8: true, fromString: true}
-
+                        ).code;
-            console.log('Minifying CSS and writing to `media/css/include.css`');
+                    } catch(e) {
-            fs.readFile(cssSource, function(err, data) {
+                        console.error('Error during minification.', e);
-                if (err) {
-                    console.warn('Error reading file: ' + cssSource);
-                }
-                var out = cleanCSS.process(data.toString());
-                fs.writeFile(src_dir + '/media/css/include.css', out, function(err) {
-                    if (err) {
-                        console.warn('Error writing `include.css` to disk.');
                     }
-                });
-            });
-
-            console.log('Minifying JS and writing to `media/js/include.js`');
-            var result = require('uglify-js').minify(src_dir + '/media/include.js', {
-                screw_ie8: true
-            });
-            fs.writeFile(src_dir + '/media/js/include.js', result.code, function(err) {
-                if (err) {
-                    console.warn('Error writing `include.js` to disk.');
                 }
+                var include_js = path.resolve(src_dir, 'media/js/include.js');
+                fs.writeFile(include_js, data, function(err) {
+                    if (err) {
+                        console.warn('Error writing `include.js` to disk.');
+                        return;
+                    }
+                    console.log('Created ' + include_js);
+                });
             });
         }
     );
+
+    build.css(src_dir, function(data) {
+        if (!raw) {
+            data = require('clean-css').process(data);
+        }
+        var include_css = path.resolve(src_dir, 'media/css/include.css');
+        fs.writeFile(include_css, data, function(err) {
+            if (err) {
+                console.warn('Error writing `include.css` to disk.');
+                return;
+            }
+            console.log('Created ' + include_css);
+        });
+    });
 }
 
 function wrap(func) {
@@ -378,4 +453,5 @@ module.exports.update = wrap(update);
 module.exports.clean = wrap(clean);
 module.exports.generate_langpacks = wrap(generate_langpacks);
 module.exports.extract_l10n = wrap(extract_l10n);
+module.exports.compile = wrap(compile);
 module.exports.build_includes = wrap(build_includes);

lib/compile.js

@@ -1,64 +0,0 @@
-#!/usr/bin/env node
-var fs = require('fs');
-var path = require('path');
-
-// nunjucks
-var compiler = require('nunjucks').compiler;
-var parser = require('nunjucks').parser;
-
-var utils = require('./utils');
-var srcdir = require('./info').src_dir();
-
-function process(folder, output_file, callback) {
-    var extensions = require('./deferparser').extensions || [];
-
-    utils.glob(folder, '.html', function(err, templates) {
-        var template_strings = (
-            '(function() {' +
-            'var templates = {};\n'
-        );
-
-        for(var i=0; i<templates.length; i++) {
-            var name = templates[i].replace(path.join(folder, '/'), '');
-            template_strings += 'templates["' + name + '"] = (function() {';
-
-            var doCompile = function() {
-                var src = fs.readFileSync(templates[i], 'utf-8');
-                var cinst = new compiler.Compiler(extensions);
-                cinst.compile(parser.parse(src, extensions));
-                template_strings += cinst.getCode();
-            };
-
-            try {
-                doCompile();
-            } catch(e) {
-                template_strings += [
-                    'return {root: function() {',
-                    'throw new Error("' + name + ' failed to compile. Check the damper for details.");',
-                    '}}'
-                ].join('\n');
-
-                console.error(e);
-            }
-
-            template_strings += '})();\n';
-        }
-
-        template_strings += (
-            'define("templates", ["nunjucks", "helpers"], function(nunjucks) {\n' +
-            '    nunjucks.env = new nunjucks.Environment([], {autoescape: true});\n' +
-            '    nunjucks.env.registerPrecompiled(templates);\n' +
-            '    nunjucks.templates = templates;\n' +
-            '    console.log("Templates loaded");\n' +
-            '    return nunjucks;\n' +
-            '});\n' +
-            '})();'
-        );
-
-        fs.writeFile(output_file, template_strings, callback);
-
-    });
-
-}
-
-module.exports.process = process;

lib/info.js

@@ -11,7 +11,7 @@ module.exports.src_dir = function() {
     }
 };
 
-module.exports.version = function() {
+var version_ = module.exports.version = function() {
     var package_json = path.resolve(__dirname, '../package.json');
     return JSON.parse(fs.readFileSync(package_json)).version;
 };
@@ -20,7 +20,7 @@ module.exports.check_version = function(src_dir, same, different, neither) {
     var existing_manifest = path.resolve(src_dir, '.commonplace');
     if (fs.existsSync(existing_manifest)) {
         var version = JSON.parse(fs.readFileSync(existing_manifest)).version;
-        var current_version = info.version();
+        var current_version = version_();
         if (version !== current_version && different) {
             different(version, current_version);
         } else if (version === current_version && same) {