Improved compilation and added a complete up-to-date check

bin/rhtmlc.js

@@ -1,29 +1,52 @@
 var raptorTemplatesCompiler = require('../compiler');
-var glob = require("glob");
 var fs = require('fs');
-var globPatterns;
-var raptorPromises = require('raptor-promises');
+var nodePath = require('path');
+var Minimatch = require('minimatch').Minimatch;
+var cwd = process.cwd();
+require('raptor-ecma/es6');
 
-var argv = require('raptor-args').createParser({
+var mmOptions = {
+    matchBase: true,
+    dot: true,
+    flipNegate: true
+};
+
+function relPath(path) {
+    if (path.startsWith(cwd)) {
+        return path.substring(cwd.length+1);
+    }
+}
+
+var args = require('raptor-args').createParser({
         '--help': {
             type: 'boolean',
             description: 'Show this help message'
         },
-        '--templates --template -t *': {
+        '--files --file -f *': {
             type: 'string[]',
-            description: 'The path to a template to compile'
+            description: 'A set of directories or files to compile'
+        },
+        '--ignore -i': {
+            type: 'string[]',
+            description: 'An ignore rule (default: --ignore "/node_modules" ".*")'
+        },
+        '--clean -c': {
+            type: 'boolean',
+            description: 'Clean all of the *.rhtml.js files'
         }
     })
     .usage('Usage: $0 <pattern> [options]')
-    .example('Compile a single template', '$0 rhtml template.rhtml')
-    .example('Compile all templates in the directory tree', '$0 rhtml **/*.rhtml')
+    .example('Compile a single template', '$0 template.rhtml')
+    .example('Compile all templates in the current directory', '$0 .')
+    .example('Compile multiple templates', '$0 template.rhtml src/ foo/')
+    .example('Delete all *.rhtml.js files in the current directory', '$0 . --clean')
     .validate(function(result) {
         if (result.help) {
             this.printUsage();
             process.exit(0);
         }
 
-        if (!result.templates || result.templates.length === 0) {
+        if (!result.files || result.files.length === 0) {
             this.printUsage();
             process.exit(1);
         }
@@ -40,64 +63,225 @@ var argv = require('raptor-args').createParser({
     })
     .parse();
 
-globPatterns = argv.templates;
-var found = {};
-var promises = [];
 
-function compile(path) {
-    if (found[path]) {
-        return;
+var ignoreRules = args.ignore;
+
+if (!ignoreRules) {
+    ignoreRules = ['/node_modules', '.*'];
+}
+
+ignoreRules = ignoreRules.filter(function (s) {
+    s = s.trim();
+    return s && !s.match(/^#/);
+});
+
+ignoreRules = ignoreRules.map(function (pattern) {
+    
+    return new Minimatch(pattern, mmOptions);
+});
+
+
+function isIgnored(path, dir, stat) {
+    if (path.startsWith(dir)) {
+        path = path.substring(dir.length);
     }
 
-    found[path] = true;
+    path = path.replace(/\\/g, '/');
 
-    var deferred = raptorPromises.defer();
+    var ignore = false;
+    var ignoreRulesLength = ignoreRules.length;
+    for (var i=0; i<ignoreRulesLength; i++) {
+        var rule = ignoreRules[i];
         
-    var outPath = path + '.js';
-    console.log('Compiling "' + path + '" to "' + outPath + '"...');
-    raptorTemplatesCompiler.compileFile(path, function(err, src) {
-        if (err) {
-            console.log('Failed to compile "' + path + '". Error: ' + (err.stack || err));
-            deferred.reject(err);
+        var match = rule.match(path);
+        
+        if (!match && stat && stat.isDirectory()) {
+            try {
+                stat = fs.statSync(path);
+            } catch(e) {}
+
+            if (stat && stat.isDirectory()) {
+                match = rule.match(path + '/');
+            }    
+        }
+        
+
+        if (match) {
+            if (rule.negate) {
+                ignore = false;
+            } else {
+                ignore = true;
+            }
+        }
+    }
+
+    return ignore;
+}
+
+function walk(files, options, done) {
+    var pending = 0;
+
+    if (!Array.isArray(files)) {
+        files = [];
+    }
+
+    var fileCallback = options.file;
+    var context = {
+        beginAsync: function() {
+            pending++;
+        },
+        endAsync: function(err) {
+            pending--;
+
+            if (err) {
+                return done(err);
+            }
+
+            if (pending === 0) {
+                done(null);
+            }
+        }
+    };
+
+    function doWalk(dir) {
+        context.beginAsync();
+        fs.readdir(dir, function(err, list) {
+            if (err) {
+                return context.endAsync(err);
+            }
+
+            if (list.length) {
+                list.forEach(function(basename) {
+                    var file = nodePath.join(dir, basename);
+
+                    context.beginAsync();
+                    fs.stat(file, function(err, stat) {
+                        if (err) {
+                            return context.endAsync(err);
+                        }
+
+                        if (!isIgnored(file, dir, stat)) {
+                            if (stat && stat.isDirectory()) {
+                                doWalk(file);
+                            } else {
+                                fileCallback(file, context);
+                            }    
+                        }
+
+                        context.endAsync();
+                    });
+                });
+            }
+
+            context.endAsync();
+        });
+    }
+
+    for (var i=0; i<files.length; i++) {
+        var file = nodePath.resolve(cwd, files[i]);
+
+        var stat = fs.statSync(file);
+
+        if (stat.isDirectory()) {
+            doWalk(file);
+        } else {
+            fileCallback(file, context);
+        }
+    }
+}
+
+if (args.clean) {
+    var deleteCount = 0;
+
+    walk(
+        args.files,
+        {
+            file: function(file, context) {
+                var basename = nodePath.basename(file);
+
+                if (basename.endsWith('.rhtml.js') || basename.endsWith('.rxml.js')) {
+                    context.beginAsync();
+                    fs.unlink(file, function(err) {
+                        if (err) {
+                            return context.endAsync(err);
+                        }
+                        deleteCount++;
+                        console.log('Deleted: ' + file);
+                        context.endAsync();
+                    });
+
+                    
+                }
+            }
+        },
+        function(err) {
+            if (deleteCount === 0) {
+                console.log('No *.rhtml.js files were found. Already clean.');
+            } else {
+                console.log('Deleted ' + deleteCount + ' file(s)');
+            }
+            
+        });
+
+} else {
+    var found = {};
+    var compileCount = 0;
+    var failed = [];
+
+    var compile = function(path, context) {
+        if (found[path]) {
             return;
         }
 
-        fs.writeFile(outPath, src, {encoding: 'utf8'}, function(err, src) {
+        found[path] = true;
+
+        var outPath = path + '.js';
+        console.log('Compiling:\n  Input: ' + relPath(path) + '\n  Output: ' + relPath(outPath) + '\n');
+        context.beginAsync();
+        raptorTemplatesCompiler.compileFile(path, function(err, src) {
             if (err) {
-                console.log('Failed to write "' + path + '". Error: ' + (err.stack || err));
-                deferred.reject(err);
+                failed.push('Failed to compile "' + path + '". Error: ' + (err.stack || err));
+                context.endAsync(err);
                 return;
             }
 
-            deferred.resolve();
+            context.beginAsync();
+            fs.writeFile(outPath, src, {encoding: 'utf8'}, function(err, src) {
+                if (err) {
+                    failed.push('Failed to write "' + path + '". Error: ' + (err.stack || err));
+                    context.endAsync(err);
+                    return;
+                }
 
+                compileCount++;
+                context.endAsync();
+
+            });
+
+            context.endAsync();
         });
-    });
+    };
 
-    return deferred.promise;
+
+    if (args.files && args.files.length) {
+        walk(
+            args.files,
+            {
+                file: function(file, context) {
+                    var basename = nodePath.basename(file);
+
+                    if (basename.endsWith('.rhtml') || basename.endsWith('.rxml')) {
+                        compile(file, context);
+                    }
+                }
+            },
+            function(err) {
+                if (compileCount === 0) {
+                    console.log('No templates found');
+                } else {
+                    console.log('Compiled ' + compileCount + ' templates(s)');
+                }
+                
+            });
+    }    
 }
-
-globPatterns.forEach(function(globPattern) {
-    var deferred = raptorPromises.defer();
-
-    glob(globPattern, function (err, files) {
-        if (err) {
-            deferred.reject(err);
-            return;
-        }
-
-        var compilePromises = files.map(compile);
-        deferred.resolve(raptorPromises.all(compilePromises));
-    });
-
-
-    promises.push(deferred.promise);
-});
-
-raptorPromises.all(promises).then(
-    function() {
-        console.log('Done!');
-    },
-    function(err) {
-        console.log('One or more templates failed to compile');
-    });

compiler/lib/Taglib.js

@@ -32,9 +32,19 @@ function Taglib(id) {
     this.helperObject = null;
     this.patternAttributes = [];
     this.importPaths = [];
+    this.inputFilesLookup = {};
 }
 
 Taglib.prototype = {
+
+    addInputFile: function(path) {
+        this.inputFilesLookup[path] = true;
+    },
+
+    getInputFiles: function() {
+        return Object.keys(this.inputFilesLookup);
+    },
+
     addAttribute: function (attribute) {
         if (attribute.namespace) {
             throw createError(new Error('"namespace" is not allowed for taglib attributes'));

compiler/lib/TaglibLookup.js

@@ -12,6 +12,7 @@ function TaglibLookup() {
     this.nestedTags = {};
     this.taglibsById = {};
     this.unresolvedAttributes = [];
+    this._inputFiles = null;
 }
 
 TaglibLookup.prototype = {
@@ -333,6 +334,30 @@ TaglibLookup.prototype = {
             throw new Error('Invalid taglib URI: ' + namespace);
         }
         return taglib.getHelperObject();
+    },
+
+    getInputFiles: function() {
+        if (!this._inputFiles) {
+            var inputFilesSet = {};
+
+            for (var taglibId in this.taglibsById) {
+                if (this.taglibsById.hasOwnProperty(taglibId)) {
+
+                    var taglibInputFiles = this.taglibsById[taglibId].getInputFiles();
+                    var len = taglibInputFiles.length;
+                    if (len) {
+                        for (var i=0; i<len; i++) {
+                            inputFilesSet[taglibInputFiles[i]] = true;
+                        }
+                    }
+                }
+            }
+
+            this._inputFiles = Object.keys(inputFilesSet); 
+        }
+
+        return this._inputFiles;
+        
     }
 };
 module.exports = TaglibLookup;

compiler/lib/TaglibXmlLoader.js

@@ -172,6 +172,7 @@ TaglibXmlLoader.prototype = {
                         if (!taglib) {
                             taglib = newTaglib;
                         }
+                        taglib.addInputFile(filePath);
                         return newTaglib;
                     },
                     'attribute': attributeHandler,

compiler/lib/TemplateCompiler.js

@@ -85,7 +85,7 @@ TemplateCompiler.prototype = {
             transformTreeHelper(rootNode);    //Run the transforms on the tree                 
         } while (this._transformerApplied);
     },
-    compile: function (xmlSrc, callback, thisObj) {
+    compile: function (src, callback, thisObj) {
         var _this = this;
         var filePath = this.path;
         var rootNode;
@@ -105,7 +105,7 @@ TemplateCompiler.prototype = {
             /*
              * First build the parse tree for the tempate
              */
-            rootNode = parser.parse(xmlSrc, filePath, this.taglibs);
+            rootNode = parser.parse(src, filePath, this.taglibs);
             //Build a parse tree from the input XML
             templateBuilder = new TemplateBuilder(this, filePath, rootNode);
             //The templateBuilder object is need to manage the compiled JavaScript output              
@@ -173,6 +173,46 @@ TemplateCompiler.prototype = {
     createTag: function () {
         var Taglib = require('./Taglib');
         return new Taglib.Tag();
+    },
+    checkUpToDate: function(sourceFile, targetFile) {
+        var fs = require('fs');
+
+        
+        var statTarget;
+
+        try {
+            statTarget = fs.statSync(targetFile);
+        } catch(e) {
+            return false;
+        }
+
+        var statSource = fs.statSync(sourceFile);
+
+        if (statSource.mtime.getTime() > statTarget.mtime.getTime()) {
+            return false;
+        }
+
+        // Now check if any of the taglib files have been modified after the target file was generated
+        
+        var taglibFiles = this.taglibs.getInputFiles();
+        var len = taglibFiles.length;
+        for (var i=0; i<len; i++) {
+            var taglibFileStat;
+            var taglibFile = taglibFiles[i];
+
+            try {
+                taglibFileStat = fs.statSync(taglibFile);
+            } catch(e) {
+                continue;
+            }
+
+            if (taglibFileStat.mtime.getTime() > statTarget.mtime.getTime()) {
+                return false;
+            }
+        }
+
+        return true;
+
     }
 };
 module.exports = TemplateCompiler;

compiler/lib/taglib-loader.js

@@ -278,6 +278,11 @@ function scanTagsDir(tagsConfigPath, tagsConfigDirname, dir, taglib) {
         var tagFile = nodePath.join(dir, childFilename, 'raptor-tag.json');
         var tagObject;
         var tag;
+        var rendererFile = nodePath.join(dir, childFilename, 'renderer.js');
+
+        // Record dependencies so that we can check if a template is up-to-date
+        taglib.addInputFile(tagFile);
+        taglib.addInputFile(rendererFile);
 
         if (fs.existsSync(tagFile)) {
             // raptor-tag.json exists in the directory, use that as the tag definition
@@ -287,7 +292,7 @@ function scanTagsDir(tagsConfigPath, tagsConfigDirname, dir, taglib) {
             taglib.addTag(tag);
         } else {
             // raptor-tag.json does *not* exist... checking for a 'renderer.js'
-            var rendererFile = nodePath.join(dir, childFilename, 'renderer.js');
+            
             if (fs.existsSync(rendererFile)) {
                 var rendererCode = fs.readFileSync(rendererFile, {encoding: 'utf8'});
                 var tagDef = tagDefFromCode.extractTagDef(rendererCode);
@@ -318,6 +323,7 @@ function load(path) {
 
     var src = fs.readFileSync(path, {encoding: 'utf8'});
     var taglib = new Taglib(path);
+    taglib.addInputFile(path);
     var dirname = nodePath.dirname(path);
 
     function handleNS(ns) {
@@ -355,6 +361,8 @@ function load(path) {
 
                 if (typeof path === 'string') {
                     path = nodePath.resolve(dirname, path);
+                    taglib.addInputFile(path);
+                    
                     tagDirname = nodePath.dirname(path);
                     if (!fs.existsSync(path)) {
                         throw new Error('Tag at path "' + path + '" does not exist. Taglib: ' + taglib.id);
@@ -368,8 +376,7 @@ function load(path) {
                     catch(e) {
                         throw new Error('Unable to parse tag JSON for tag at path "' + path + '"');
                     }
-                }
-                else {
+                } else {
                     tagDirname = dirname; // Tag is in the same taglib file
                     tagObject = path;
                     path = '<' + tagName + '> tag in ' + taglib.id;

package.json

@@ -1,56 +1,57 @@
 {
-    "name": "raptor-templates",
-    "description": "Raptor Templates",
-    "keywords": [
-        "templating",
-        "template",
-        "async",
-        "streaming"
-    ],
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/raptorjs3/raptor-templates.git"
-    },
-    "scripts": {
-        "test": "node_modules/.bin/mocha --ui bdd --reporter spec ./test"
-    },
-    "author": "Patrick Steele-Idem <pnidem@gmail.com>",
-    "maintainers": [
-        "Patrick Steele-Idem <pnidem@gmail.com>"
-    ],
-    "dependencies": {
-        "raptor-detect": "^0.2.0-beta",
-        "raptor-logging": "^0.2.0-beta",
-        "raptor-strings": "^0.2.0-beta",
-        "raptor-regexp": "^0.2.0-beta",
-        "raptor-util": "^0.2.0-beta",
-        "raptor-arrays": "^0.2.0-beta",
-        "raptor-json": "^0.2.0-beta",
-        "raptor-modules": "^0.2.0-beta",
-        "raptor-render-context": "^0.2.0-beta",
-        "raptor-data-providers": "^0.2.0-beta",
-        "raptor-xml": "^0.2.0-beta",
-        "raptor-objects": "^0.2.0-beta",
-        "raptor-ecma": "^0.2.0-beta",
-        "raptor-files": "^0.2.0-beta",
-        "htmlparser2": "~3.5.1",
-        "char-props": "~0.1.5",
-        "raptor-promises": "^0.2.0-beta",
-        "glob": "^3.2.9",
-        "raptor-args": "^0.1.9-beta"
-    },
-    "devDependencies": {
-        "mocha": "~1.15.1",
-        "chai": "~1.8.1",
-        "raptor-cache": "^0.2.0-beta"
-    },
-    "license": "Apache License v2.0",
-    "bin": {
-        "rhtmlc": "bin/rhtmlc"
-    },
-    "main": "runtime/lib/raptor-templates.js",
-    "publishConfig": {
-        "registry": "https://registry.npmjs.org/"
-    },
-    "version": "0.2.23-beta"
+  "name": "raptor-templates",
+  "description": "Raptor Templates",
+  "keywords": [
+    "templating",
+    "template",
+    "async",
+    "streaming"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/raptorjs3/raptor-templates.git"
+  },
+  "scripts": {
+    "test": "node_modules/.bin/mocha --ui bdd --reporter spec ./test"
+  },
+  "author": "Patrick Steele-Idem <pnidem@gmail.com>",
+  "maintainers": [
+    "Patrick Steele-Idem <pnidem@gmail.com>"
+  ],
+  "dependencies": {
+    "raptor-detect": "^0.2.0-beta",
+    "raptor-logging": "^0.2.0-beta",
+    "raptor-strings": "^0.2.0-beta",
+    "raptor-regexp": "^0.2.0-beta",
+    "raptor-util": "^0.2.0-beta",
+    "raptor-arrays": "^0.2.0-beta",
+    "raptor-json": "^0.2.0-beta",
+    "raptor-modules": "^0.2.0-beta",
+    "raptor-render-context": "^0.2.0-beta",
+    "raptor-data-providers": "^0.2.0-beta",
+    "raptor-xml": "^0.2.0-beta",
+    "raptor-objects": "^0.2.0-beta",
+    "raptor-ecma": "^0.2.0-beta",
+    "raptor-files": "^0.2.0-beta",
+    "htmlparser2": "~3.5.1",
+    "char-props": "~0.1.5",
+    "raptor-promises": "^0.2.0-beta",
+    "glob": "^3.2.9",
+    "raptor-args": "^0.1.9-beta",
+    "minimatch": "^0.2.14"
+  },
+  "devDependencies": {
+    "mocha": "~1.15.1",
+    "chai": "~1.8.1",
+    "raptor-cache": "^0.2.0-beta"
+  },
+  "license": "Apache License v2.0",
+  "bin": {
+    "rhtmlc": "bin/rhtmlc"
+  },
+  "main": "runtime/lib/raptor-templates.js",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/"
+  },
+  "version": "0.2.23-beta"
 }

runtime/lib/loader.js

@@ -1,14 +1,16 @@
 var nodePath = require('path');
 var fs = require('fs');
 var Module = require('module').Module;
-var compiler = require('../../compiler');
+var raptorTemplatesCompiler = require('../../compiler');
 
 function loadSource(templatePath, compiledSrc) {
     var templateModulePath = templatePath + '.js';
+
     var templateModule = new Module(templateModulePath, module);
     templateModule.paths = Module._nodeModulePaths(nodePath.dirname(templateModulePath));
     templateModule.filename = templateModulePath;
 
+
     templateModule._compile(
         compiledSrc,
         templateModulePath);
@@ -17,10 +19,21 @@ function loadSource(templatePath, compiledSrc) {
 }
 
 module.exports = function load(templatePath) {
+    var targetFile = templatePath + '.js';
+    var compiler = raptorTemplatesCompiler.createCompiler(templatePath);
+    var isUpToDate = compiler.checkUpToDate(templatePath, targetFile);
+    if (isUpToDate) {
+        return require(targetFile);
+    }
+
 	var templateSrc = fs.readFileSync(templatePath, {encoding: 'utf8'});
-	var compiledSrc = compiler.compile(templateSrc, templatePath);
+	var compiledSrc = compiler.compile(templateSrc);
+
     // console.log('Compiled code for "' + templatePath + '":\n' + compiledSrc);
-	return loadSource(templatePath, compiledSrc);
+     
+    fs.writeFileSync(targetFile, compiledSrc, {encoding: 'utf8'});
+
+    return require(targetFile);
 };
 
 module.exports.loadSource = loadSource;

test/.gitignore

@@ -1,2 +1,4 @@
+*.rhtml.js
+*.rxml.js
 *.actual.js
 *.actual.html

test/test-project/rhtml-templates/simple.rhtml.js

@@ -1,22 +1,37 @@
- $rtmpl("simple", function (templating) {
-     var empty = templating.e,
-         notEmpty = templating.ne,
-         forEach = templating.f;
-     return function (data, context) {
-         var write = context.w,
-             rootClass = data.rootClass,
-             colors = data.colors,
-             message = data.message;
-         write('<div class="hello-world ', rootClass, '">', message, '</div>');
-         if (notEmpty(colors)) {
-             write('<ul>');
-             forEach(colors, function (color) {
-                 write('<li class="color">', color, '</li>');
-             });
-             write('</ul>');
-         }
-         if (empty(colors)) {
-             write('<div>No colors!</div>');
-         }
-     }
- });
+module.exports = function create(helpers) {
+  var empty = helpers.e,
+      notEmpty = helpers.ne,
+      escapeXmlAttr = helpers.xa,
+      escapeXml = helpers.x,
+      forEach = helpers.f;
+
+  return function render(data, context) {
+    var rootClass=data.rootClass;
+
+    var colors=data.colors;
+
+    var message=data.message;
+
+    context.w('<div class="hello-world ')
+      .w(escapeXmlAttr(rootClass))
+      .w('">')
+      .w(escapeXml(message))
+      .w('</div>');
+
+    if (notEmpty(colors)) {
+      context.w('<ul>');
+
+      forEach(colors, function(color) {
+        context.w('<li class="color">')
+          .w(escapeXml(color))
+          .w('</li>');
+      });
+
+      context.w('</ul>');
+    }
+
+    if (empty(colors)) {
+      context.w('<div>No colors!</div>');
+    }
+  };
+}

test/test-project/rxml-templates/simple.rxml.js

@@ -1,22 +1,42 @@
- $rtmpl("simple", function (templating) {
-     var empty = templating.e,
-         notEmpty = templating.ne,
-         forEach = templating.f;
-     return function (data, context) {
-         var write = context.w,
-             rootClass = data.rootClass,
-             colors = data.colors,
-             message = data.message;
-         write('<div class="hello-world ', rootClass, '">', message, '</div>');
-         if (notEmpty(colors)) {
-             write('<ul>');
-             forEach(colors, function (color) {
-                 write('<li class="color">', color, '</li>');
-             });
-             write('</ul>');
-         }
-         if (empty(colors)) {
-             write('<div>No colors!</div>');
-         }
-     }
- });
+module.exports = function create(helpers) {
+  var empty = helpers.e,
+      notEmpty = helpers.ne,
+      hello_renderer = require("../hello-renderer"),
+      escapeXmlAttr = helpers.xa,
+      escapeXml = helpers.x,
+      forEach = helpers.f;
+
+  return function render(data, context) {
+    var rootClass = data.rootClass,
+        colors = data.colors,
+        message = data.message;
+
+    helpers.t(context, 
+      hello_renderer,
+      {
+        "name": "World"
+      });
+
+    context.w('<div class="hello-world ')
+      .w(escapeXmlAttr(rootClass))
+      .w('">')
+      .w(escapeXml(message))
+      .w('</div>');
+
+    if (notEmpty(colors)) {
+      context.w('<ul>');
+
+      forEach(colors, function(color) {
+        context.w('<li class="color">')
+          .w(escapeXml(color))
+          .w('</li>');
+      });
+
+      context.w('</ul>');
+    }
+
+    if (empty(colors)) {
+      context.w('<div>No colors!</div>');
+    }
+  };
+}