Fixes #205 - Marko v3: Provide full control over whitespace

compiler/Builder.js

@@ -352,10 +352,14 @@ class Builder {
         return new TemplateRoot({body});
     }
 
-    text(argument, escape) {
+    text(argument, escape, preserveWhitespace) {
+        if (typeof argument === 'object' && !(argument instanceof Node)) {
+            var def = arguments[0];
+            return new Text(def);
+        }
         argument = makeNode(argument);
 
-        return new Text({argument, escape});
+        return new Text({argument, escape, preserveWhitespace});
     }
 
     thisExpression() {

compiler/CodeGenerator.js

@@ -327,22 +327,11 @@ class Generator {
     }
 
     addWriteLiteral(value) {
-        let lastWrite = this._bufferedWrites ?
-            this._bufferedWrites[this._bufferedWrites.length-1] :
-            null;
-
-        if (lastWrite instanceof Literal) {
-            lastWrite.value += value;
-            return;
+        if (!(value instanceof Literal)) {
+            value = new Literal({value});
         }
 
-        let output = new Literal({value});
-
-        if (!this._bufferedWrites) {
-            this._bufferedWrites = [output];
-        } else {
-            this._bufferedWrites.push(output);
-        }
+        this.addWrite(value);
     }
 
     addWrite(output) {
@@ -355,6 +344,10 @@ class Generator {
                 lastWrite.value += output.value;
                 return;
             }
+        } else {
+            if (!(output instanceof Node)) {
+                throw new Error('Invalid write: ' + JSON.stringify(output, null, 2));
+            }
         }
 
         if (!this._bufferedWrites) {

compiler/CompileContext.js

@@ -58,6 +58,7 @@ class CompileContext {
         this._flags = {};
         this._errors = [];
         this._macros = null;
+        this._preserveWhitespace = null;
     }
 
     getPosInfo(pos) {
@@ -144,6 +145,21 @@ class CompileContext {
         return this._staticCode;
     }
 
+    getTagDef(tagName) {
+        var taglibLookup = this.taglibLookup;
+
+        if (typeof tagName === 'string') {
+            return taglibLookup.getTag(tagName);
+        } else {
+            let elNode = tagName;
+            if (elNode.tagDef) {
+                return elNode.tagDef;
+            }
+
+            return taglibLookup.getTag(elNode.tagName);
+        }
+    }
+
     createNodeForEl(tagName, attributes, argument, openTagOnly, selfClosed) {
         var elDef;
         var builder = this.builder;
@@ -275,6 +291,14 @@ class CompileContext {
         var templateVar = this.addStaticVar(removeExt(relativePath), loadFunctionCall);
         return templateVar;
     }
+
+    setPreserveWhitespace(preserveWhitespace) {
+        this._preserveWhitespace = preserveWhitespace;
+    }
+
+    isPreserveWhitespace() {
+        return this._preserveWhitespace === true;
+    }
 }
 
 module.exports = CompileContext;

compiler/HtmlJsParser.js

@@ -3,7 +3,7 @@ var htmljs = require('htmljs-parser');
 
 class HtmlJsParser {
     parse(src, handlers) {
-        var parser = this.parser = htmljs.createParser({
+        var listeners = {
             ontext(event) {
                 handlers.handleCharacters(event.text);
             },
@@ -55,20 +55,22 @@ class HtmlJsParser {
             },
 
             onerror(event) {
-                // Error
+                handlers.handleError(event);
             }
-        });
+        };
+
+        var options = {
+            parserStateProvider(event) {
+                if (event.type === 'opentag') {
+                    return handlers.getParserStateForTag(event);
+                }
+            }
+        };
+
+        var parser = this.parser = htmljs.createParser(listeners, options);
 
         parser.parse(src);
     }
-
-    enterParsedTextContentState() {
-        this.parser.enterParsedTextContentState();
-    }
-
-    enterStaticTextContentState() {
-        this.parser.enterStaticTextContentState();
-    }
 }
 
 module.exports = HtmlJsParser;

compiler/Parser.js

@@ -2,15 +2,11 @@
 var ok = require('assert').ok;
 
 var COMPILER_ATTRIBUTE_HANDLERS = {
-    whitespace: function(attr, compilerOptions) {
-        if (attr.value === 'preserve') {
-            compilerOptions.preserveWhitespace = true;
-        }
+    'preserve-whitespace': function(attr, context) {
+        context.setPreserveWhitespace(true);
     },
-    comments: function(attr, compilerOptions) {
-        if (attr.value === 'preserve') {
-            compilerOptions.preserveComments = true;
-        }
+    'preserve-comments': function(attr, context) {
+        context.setPreserveComments(true);
     }
 };
 
@@ -70,7 +66,8 @@ class Parser {
         if (this.prevTextNode && this.prevTextNode.isLiteral()) {
             this.prevTextNode.appendText(text);
         } else {
-            this.prevTextNode = builder.text(builder.literal(text));
+            var escape = false;
+            this.prevTextNode = builder.text(builder.literal(text), escape);
             this.prevTextNode.pos = text.pos;
             this.parentNode.appendChild(this.prevTextNode);
         }
@@ -85,18 +82,21 @@ class Parser {
         var argument = el.argument; // e.g. For <for(color in colors)>, argument will be "color in colors"
 
         if (tagName === 'compiler-options') {
-            var compilerOptions = this.compilerOptions;
-
             attributes.forEach(function (attr) {
                 let attrName = attr.name;
-                let attrValue = attr.value;
-                let handler = COMPILER_ATTRIBUTE_HANDLERS[attrValue];
+                let handler = COMPILER_ATTRIBUTE_HANDLERS[attrName];
 
                 if (!handler) {
-                    throw new Error('Invalid Marko compiler option: ' + attrName + ', Allowed: ' + Object.keys(COMPILER_ATTRIBUTE_HANDLERS));
+                    context.addError({
+                        code: 'ERR_INVALID_COMPILER_OPTION',
+                        message: 'Invalid Marko compiler option of "' + attrName + '". Allowed: ' + Object.keys(COMPILER_ATTRIBUTE_HANDLERS).join(', '),
+                        pos: el.pos,
+                        node: el
+                    });
+                    return;
                 }
 
-                handler(attr, compilerOptions);
+                handler(attr, context);
             });
 
             return;
@@ -138,11 +138,7 @@ class Parser {
         if (node.tagDef) {
             var body = tagDef.body;
             if (body) {
-                if (body === 'parsed-text') {
-                    this.parserImpl.enterParsedTextContentState();
-                } else if (body === 'static-text') {
-                    this.parserImpl.enterStaticTextContentState();
-                }
+
             }
         }
 
@@ -185,15 +181,65 @@ class Parser {
         var parsedExpression = parseExpression(expression);
 
         var builder = this.context.builder;
+        var preserveWhitespace = true;
 
-        var text = builder.text(parsedExpression, escape);
+        var text = builder.text(parsedExpression, escape, preserveWhitespace);
         this.parentNode.appendChild(text);
     }
 
+    handleError(event) {
+        this.context.addError({
+            message: event.message,
+            code: event.code,
+            pos: event.pos,
+            endPos: event.endPos
+        });
+    }
+
     get parentNode() {
         var last = this.stack[this.stack.length-1];
         return last.node;
     }
+
+    getParserStateForTag(el) {
+        var attributes = el.attributes;
+
+        for (var i=0; i<attributes.length; i++) {
+            var attr = attributes[i];
+            var attrName = attr.name;
+            if (attrName === 'marko-body') {
+                var parseMode;
+
+                if (attr.literalValue) {
+                    parseMode = attr.literalValue;
+                }
+
+                if (parseMode === 'static-text' ||
+                    parseMode === 'parsed-text' ||
+                    parseMode === 'html') {
+                    return parseMode;
+                } else {
+                    this.context.addError({
+                        message: 'Value for "marko-body" should be one of the following: "static-text", "parsed-text", "html"',
+                        code: 'ERR_INVALID_ATTR'
+                    });
+                    return;
+                }
+            }
+        }
+
+        var tagName = el.tagName;
+        var tagDef = this.context.getTagDef(tagName);
+
+        if (tagDef) {
+            var body = tagDef.body;
+            if (body) {
+                return body; // 'parsed-text' | 'static-text' | 'html'
+            }
+        }
+
+        return null; // Default parse state
+    }
 }
 
 module.exports = Parser;

compiler/ast/Assignment.js

@@ -39,6 +39,13 @@ class Assignment extends Node {
     isCompoundExpression() {
         return true;
     }
+
+    /**
+     * "noOutput" should be true if the Node.js does not result in any HTML or Text output
+     */
+    get noOutput() {
+        return !(this.body && this.body.length);
+    }
 }
 
 module.exports = Assignment;

compiler/ast/Node.js

@@ -15,6 +15,7 @@ class Node {
         this._codeGeneratorFuncs = null;
         this._flags = {};
         this._transformersApplied = {};
+        this._preserveWhitespace = null;
         this.data = {};
     }
 
@@ -79,6 +80,7 @@ class Node {
         delete result._flags;
         delete result.data;
         delete result.tagDef;
+        delete result._preserveWhitespace;
         return result;
     }
 
@@ -165,6 +167,19 @@ class Node {
     get parentNode() {
         return this.container && this.container.node;
     }
+
+    setPreserveWhitespace(isPreserved) {
+        this._preserveWhitespace = isPreserved;
+    }
+
+    isPreserveWhitespace() {
+        var preserveWhitespace = this._preserveWhitespace;
+        if (preserveWhitespace == null) {
+            preserveWhitespace = this.tagDef && this.tagDef.preserveWhitespace;
+        }
+
+        return preserveWhitespace === true;
+    }
 }
 
 module.exports = Node;

compiler/ast/Text.js

@@ -1,9 +1,15 @@
 'use strict';
 
+var ok = require('assert').ok;
 var Node = require('./Node');
 var Literal = require('./Literal');
+var escapeXml = require('raptor-util/escapeXml');
 
 function trim(textNode) {
+    if (textNode.preserveWhitespace === true) {
+        return;
+    }
+
     var text = textNode.argument.value;
     var isFirst = textNode.isFirst;
     var isLast = textNode.isLast;
@@ -28,10 +34,13 @@ class Text extends Node {
     constructor(def) {
         super('Text');
         this.argument = def.argument;
-        this.escape = def.escape;
+        this.escape = def.escape !== false;
         this.normalized = false;
         this.isFirst = false;
         this.isLast = false;
+        this.preserveWhitespace = def.preserveWhitespace === true;
+
+        ok(this.argument, 'Invalid argument');
     }
 
     isLiteral() {
@@ -39,7 +48,9 @@ class Text extends Node {
     }
 
     generateHtmlCode(codegen) {
-        this.normalizeText();
+        this.normalizeText(codegen);
+
+
 
         var argument = this.argument;
         var escape = this.escape !== false;
@@ -48,6 +59,10 @@ class Text extends Node {
             if (!argument.value) {
                 return;
             }
+
+            if (escape === true) {
+                argument.value = escapeXml(argument.value.toString());
+            }
         } else {
             let builder = codegen.builder;
 
@@ -66,13 +81,15 @@ class Text extends Node {
     }
 
     normalizeText(codegen) {
-        if (this.normalized) {
+        if (this.normalized || codegen.context.isPreserveWhitespace() || this.preserveWhitespace === true) {
             return;
         }
 
         var parentNode = this.parentNode;
-        if (parentNode && parentNode.tagDef && parentNode.tagDef.preserveWhitespace) {
-            return;
+        if (parentNode) {
+            if (parentNode.isPreserveWhitespace()) {
+                return;
+            }
         }
 
         var container = this.container;
@@ -96,7 +113,9 @@ class Text extends Node {
             }
 
             if (curChild.type === 'Text' && curChild.isLiteral()) {
-                if (currentTextLiteral) {
+                if (currentTextLiteral &&
+                        currentTextLiteral.preserveWhitespace === curChild.preserveWhitespace &&
+                        currentTextLiteral.escape === curChild.escape) {
                     currentTextLiteral.argument.value += curChild.argument.value;
                     curChild.detach();
                 } else {

compiler/ast/Vars.js

@@ -16,7 +16,7 @@ class Vars extends Node {
         var isStatement = this.statement;
         var body = this.body;
         var selfInvoking = this.isFlagSet('selfInvoking');
-        var hasBody = (body && body.array && body.array.length > 0);
+        var hasBody = this.body && this.body.length;
 
         if(!selfInvoking && hasBody) {
             this.setFlag('selfInvoking');
@@ -59,6 +59,13 @@ class Vars extends Node {
     walk(walker) {
         this.argument = walker.walk(this.argument);
     }
+
+    /**
+     * "noOutput" should be true if the Node.js does not result in any HTML or Text output
+     */
+    get noOutput() {
+        return !(this.body && this.body.length);
+    }
 }
 
 module.exports = Vars;

taglibs/core/core-transformer.js

@@ -77,6 +77,11 @@ var coreAttrHandlers = [
                 node.addDynamicAttributes(attr.value);
             }
         }
+    ],
+    [
+        'marko-preserve-whitespace', function(attr, node) {
+            node.setPreserveWhitespace(true);
+        }
     ]
 ];
 
@@ -104,6 +109,8 @@ coreAttrHandlers.forEach(function(attrHandler) {
 var attributeTransformers = AttributeTransformer.prototype;
 
 module.exports = function transform(el, context) {
+    el.removeAttribute('marko-body'); // This attribute is handled at parse time. We can just remove it now
+
     var attributeTransfomer;
     var node = el;
 

test/fixtures/compiler/autotest/template-init/expected.js

@@ -4,7 +4,7 @@ function create(__helpers) {
       notEmpty = __helpers.ne,
       escapeXml = __helpers.x;
 
-  var name = 'Frank';
+  var name = '${name}<div if(foo)></div>';
 
   return function render(data, out) {
     out.w(" Hello " +

test/fixtures/compiler/autotest/template-init/template.marko

@@ -1,5 +1,5 @@
 <template-init>
-var name = 'Frank';
+var name = '${name}<div if(foo)></div>';
 </template-init>
 
 Hello ${name}!

test/fixtures/render/autotest-pending/text-replacement/template.marko

@@ -1,3 +0,0 @@
-<var name="person" value="data.person"/>
-
-Hello $person.name. You are from ${person.address.city}, $person.address.state Zero: ${data.zero}

test/fixtures/render/autotest-pending/whitespace4/template.marko

@@ -1,4 +0,0 @@
-<compiler-options whitespace="preserve" />
-A
-B
-C

test/fixtures/render/autotest/body-placeholder-literal-escaped/expected.html

@@ -0,0 +1 @@
+<div></div>

test/fixtures/render/autotest/body-placeholder-literal-escaped/template.marko

@@ -0,0 +1 @@
+${"<div></div>"}

test/fixtures/render/autotest/body-placeholder-literal-escaped/test.js

@@ -0,0 +1,7 @@
+exports.templateData = {
+    "myAttrs": {
+        "style": "background-color: #FF0000; <test>",
+        "class": "my-div",
+        "checked": true
+    }
+};

test/fixtures/render/autotest/body-placeholder-literal-unescaped-escaped/expected.html

@@ -0,0 +1 @@
+&lt;div>&lt;/div> <div></div>

test/fixtures/render/autotest/body-placeholder-literal-unescaped-escaped/template.marko

@@ -0,0 +1 @@
+${"<div></div>"} $!{"<div></div>"}

test/fixtures/render/autotest/body-placeholder-literal-unescaped-escaped/test.js

@@ -0,0 +1,7 @@
+exports.templateData = {
+    "myAttrs": {
+        "style": "background-color: #FF0000; <test>",
+        "class": "my-div",
+        "checked": true
+    }
+};

test/fixtures/render/autotest/body-placeholder-literal-unescaped/expected.html

@@ -0,0 +1 @@
+<div></div>

test/fixtures/render/autotest/body-placeholder-literal-unescaped/template.marko

@@ -0,0 +1 @@
+$!{"<div></div>"}

test/fixtures/render/autotest/body-placeholder-literal-unescaped/test.js

@@ -0,0 +1,7 @@
+exports.templateData = {
+    "myAttrs": {
+        "style": "background-color: #FF0000; <test>",
+        "class": "my-div",
+        "checked": true
+    }
+};

test/fixtures/render/autotest/compiler-option-preserve-whitespace/expected.html

@@ -1,4 +1,4 @@
 
 A
-B
+  B
 C

test/fixtures/render/autotest/compiler-option-preserve-whitespace/template.marko

@@ -0,0 +1,4 @@
+<compiler-options preserve-whitespace />
+A
+  B
+C

test/fixtures/render/autotest/marko-body-attr-static-text/expected.html

@@ -0,0 +1 @@
+<div><span if(foo)> Hello ${THIS IS NOT VALID}! </span></div>

test/fixtures/render/autotest/marko-body-attr-static-text/template.marko

@@ -0,0 +1,5 @@
+<div marko-body="static-text">
+    <span if(foo)>
+            Hello ${THIS IS NOT VALID}!
+    </span>
+</div>

test/fixtures/render/autotest/marko-body-attr-static-text/test.js

@@ -0,0 +1,7 @@
+exports.templateData = {
+    "myAttrs": {
+        "style": "background-color: #FF0000; <test>",
+        "class": "my-div",
+        "checked": true
+    }
+};

test/fixtures/render/autotest/marko-preserve-whitespace-attr/expected.html

@@ -0,0 +1,4 @@
+<div>
+    Hello
+        World
+</div>

test/fixtures/render/autotest/marko-preserve-whitespace-attr/template.marko

@@ -0,0 +1,4 @@
+<div marko-preserve-whitespace>
+    Hello
+        World
+</div>

test/fixtures/render/autotest/marko-preserve-whitespace-attr/test.js

@@ -0,0 +1,7 @@
+exports.templateData = {
+    "myAttrs": {
+        "style": "background-color: #FF0000; <test>",
+        "class": "my-div",
+        "checked": true
+    }
+};

test/fixtures/render/autotest/text-replacement/template.marko

@@ -0,0 +1,3 @@
+<var person=data.person/>
+
+Hello ${person.name}. You are from ${person.address.city}, ${person.address.state} Zero: ${data.zero}

test/fixtures/render/autotest/whitespace-contentplaceholder-literal-string/expected.html

@@ -0,0 +1 @@
+WHITE    SPACE

test/fixtures/render/autotest/whitespace-contentplaceholder-literal-string/template.marko

@@ -0,0 +1 @@
+${"WHITE    SPACE"}

test/fixtures/render/autotest/whitespace/expected.html

@@ -1,7 +1,7 @@
 BEGIN  this whitespace   should be retained   END test hello Long paragraph of text should retain spacing between lines. <ul><li>One</li><li>Two</li></ul><a href="Test">Hello World!</a><pre>
-   begin      end     
+   begin      end
 </pre><div>
-   begin      end     
+   begin      end
 </div><div>
-   begin      end     
+   begin      end
 </div>begin end

test/fixtures/render/autotest/whitespace/template.marko

@@ -1,7 +1,7 @@
-${"BEGIN  this whitespace   should be retained   END"} 
+${"BEGIN  this whitespace   should be retained   END"}
 
 test <!-- text should be normalized to one space -->
-${"hello"} 
+${"hello"}
 
 Long paragraph of text
 
@@ -19,18 +19,18 @@ should   <!-- This whitespace should be normalized -->  retain spacing between l
 </a>
 
 <pre>
-   begin <!-- this whitespace should not be normalized -->     end     
+   begin <!-- this whitespace should not be normalized -->     end
 </pre>
 
-<div c-space="preserve">
-   begin <!-- this whitespace should not be normalized -->     end     
+<div marko-preserve-whitespace>
+   begin <!-- this whitespace should not be normalized -->     end
 </div>
 
-<div c-space="preserve">
-   begin <!-- this whitespace should not be normalized -->     end     
+<div marko-preserve-whitespace>
+   begin <!-- this whitespace should not be normalized -->     end
 </div>
-<if test="true">begin <!-- this whitespace should be preserved -->end</if>
-<!-- 
+<if(true)>begin <!-- this whitespace should be preserved -->end</if>
+<!--
 - In not "xml:space" === "preserve":
     - newline followed by whitespace should be removed
     - More than one whitespace should be normalized into a single space (or new line character?)

test/fixtures/render/autotest/whitespace/test.js

@@ -0,0 +1 @@
+exports.templateData = {};