lexer: implement variable interpolation in text

Tokenizes a variable token between tag interpolation tokens
if a #{variable} reference is found inside pipeless text.
Result is equivalent to writing #[#{variable}].

This reverts and improves commit 859c80f.

packages/lexer/index.js

@@ -476,6 +476,8 @@ Lexer.prototype = {
     var indexOfEnd = this.interpolated ? value.indexOf(']') : -1;
     var indexOfStart = this.interpolationAllowed ? value.indexOf('#[') : -1;
     var indexOfEscaped = this.interpolationAllowed ? value.indexOf('\\#[') : -1;
+    let matchOfVarRef = /(\\)?#{(\w+)}/.exec(value)
+    let indexOfVarRef = matchOfVarRef? matchOfVarRef.index : Infinity;
 
     if (indexOfEnd === -1) indexOfEnd = Infinity;
     if (indexOfStart === -1) indexOfStart = Infinity;
@@ -484,7 +486,8 @@ Lexer.prototype = {
     if (
       indexOfEscaped !== Infinity &&
       indexOfEscaped < indexOfEnd &&
-      indexOfEscaped < indexOfStart
+      indexOfEscaped < indexOfStart &&
+      indexOfEscaped < indexOfVarRef
     ) {
       prefix = prefix + value.substring(0, indexOfEscaped) + '#[';
       return this.addText(
@@ -497,7 +500,8 @@ Lexer.prototype = {
     if (
       indexOfStart !== Infinity &&
       indexOfStart < indexOfEnd &&
-      indexOfStart < indexOfEscaped
+      indexOfStart < indexOfEscaped &&
+      indexOfStart < indexOfVarRef
     ) {
       tok = this.tok(type, prefix + value.substring(0, indexOfStart));
       this.incrementColumn(prefix.length + indexOfStart + escaped);
@@ -533,7 +537,8 @@ Lexer.prototype = {
     if (
       indexOfEnd !== Infinity &&
       indexOfEnd < indexOfStart &&
-      indexOfEnd < indexOfEscaped
+      indexOfEnd < indexOfEscaped &&
+      indexOfEnd < indexOfVarRef
     ) {
       if (prefix + value.substring(0, indexOfEnd)) {
         this.addText(type, value.substring(0, indexOfEnd), prefix);
@@ -543,6 +548,40 @@ Lexer.prototype = {
       return;
     }
 
+    if (indexOfVarRef !== Infinity) {
+      if (matchOfVarRef[1]) {
+        // escaped: \#{
+        prefix = prefix + value.substring(0, indexOfVarRef) + '#{';
+        return this.addText(
+          type,
+          value.substring(indexOfVarRef + 3),
+          prefix,
+          escaped + 1
+        );
+      }
+      let before = value.substr(0, indexOfVarRef);
+      if (prefix || before) {
+        before = prefix + before;
+        tok = this.tok(type, before);
+        this.incrementColumn(before.length + escaped);
+        this.tokens.push(this.tokEnd(tok));
+      }
+
+      tok = this.tok('start-interpolation');
+      this.incrementColumn(2);
+      this.tokens.push(this.tokEnd(tok));
+
+      tok = this.tok('variable', matchOfVarRef[2]);
+      this.tokens.push(tok);
+      this.incrementColumn(matchOfVarRef[2].length);
+
+      tok = this.tok('end-interpolation');
+      this.incrementColumn(1);
+      this.tokens.push(this.tokEnd(tok));
+
+      value = value.substr(value.indexOf('}') + 1);
+    }
+
     value = prefix + value;
     tok = this.tok(type, value);
     this.incrementColumn(value.length + escaped);

packages/lexer/test/__snapshots__/index.test.js.snap

@@ -9194,7 +9194,7 @@ exports[`filters.verbatim.pg 1`] = `
   {
     "loc": {
       "end": {
-        "column": 41,
+        "column": 35,
         "line": 5,
       },
       "filename": "filters.verbatim.pg",
@@ -9204,7 +9204,61 @@ exports[`filters.verbatim.pg 1`] = `
       },
     },
     "type": "text",
-    "val": "with no #[b interpolation] at #{all}",
+    "val": "with no #[b interpolation] at ",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 37,
+        "line": 5,
+      },
+      "filename": "filters.verbatim.pg",
+      "start": {
+        "column": 35,
+        "line": 5,
+      },
+    },
+    "type": "start-interpolation",
+  },
+  {
+    "loc": {
+      "filename": "filters.verbatim.pg",
+      "start": {
+        "column": 37,
+        "line": 5,
+      },
+    },
+    "type": "variable",
+    "val": "all",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 41,
+        "line": 5,
+      },
+      "filename": "filters.verbatim.pg",
+      "start": {
+        "column": 40,
+        "line": 5,
+      },
+    },
+    "type": "end-interpolation",
+  },
+  {
+    "loc": {
+      "end": {
+        "column": 41,
+        "line": 5,
+      },
+      "filename": "filters.verbatim.pg",
+      "start": {
+        "column": 41,
+        "line": 5,
+      },
+    },
+    "type": "text",
+    "val": "",
   },
   {
     "loc": {