Use classes after all

src/lexer.js

@@ -1,69 +1,59 @@
-const tokenTypes = {
-  OPAREN: '(',
-  CPAREN: ')',
-  KEYWORD: 'keyword',
-  QUOTE: '"',
-  LITERAL: 'literal',
-  ATTRIBUTE: 'attribute',
-  EOF: 'eof',
-}
+const TokenStream = require('./tokenStream')
+const tokenTypes = require('./tokenTypes')
 
-const scan = source => {
-  let pos = 0
-  let line = 1
-  let tokens = []
+module.exports = class Lexer {
+  scan(source) {
+    let pos = 0
+    let line = 1
+    let tokens = new TokenStream()
 
-  while (pos < source.length) {
-    if (source[pos].match(/\(/)) {
-      tokens.push({
-        type: tokenTypes.OPAREN,
-        line: line,
-      })
-      pos++
-    } else if (source[pos].match(/\)/)) {
-      tokens.push({
-        type: tokenTypes.CPAREN,
-        line: line,
-      })
-      pos++
-    } else if (source[pos].match(/['"]/)) {
-      tokens.push({
-        type: tokenTypes.QUOTE,
-        line: line,
-      })
-      pos++
-    } else if (source[pos].match(/:/)) {
-      let value = /:([^()'"\s]+)/.exec(source)[1].trim()
-      tokens.push({
-        type: tokenTypes.ATTRIBUTE,
-        line: line,
-        value: value,
-      })
-      pos += value.length + 1 // the +1 is to account for the colon
-    } else if (source[pos].match(/\n/)) {
-      line++
-      pos++
-    } else if (source[pos].match(/\s/)) {
-      pos++
-    } else {
-      let value = /[^()"':]+/.exec(source.slice(pos))[0]
-      tokens.push({
-        type: tokenTypes.LITERAL,
-        line: line,
-        value: value.trim(),
-      })
-      pos += value.length
+    while (pos < source.length) {
+      if (source[pos].match(/\(/)) {
+        tokens.push({
+          type: tokenTypes.OPAREN,
+          line: line,
+        })
+        pos++
+      } else if (source[pos].match(/\)/)) {
+        tokens.push({
+          type: tokenTypes.CPAREN,
+          line: line,
+        })
+        pos++
+      } else if (source[pos].match(/['"]/)) {
+        tokens.push({
+          type: tokenTypes.QUOTE,
+          line: line,
+        })
+        pos++
+      } else if (source[pos].match(/:/)) {
+        let value = /:([^()'"\s]+)/.exec(source)[1].trim()
+        tokens.push({
+          type: tokenTypes.ATTRIBUTE,
+          line: line,
+          value: value,
+        })
+        pos += value.length + 1 // the +1 is to account for the colon
+      } else if (source[pos].match(/\n/)) {
+        line++
+        pos++
+      } else if (source[pos].match(/\s/)) {
+        pos++
+      } else {
+        let value = /[^()"':]+/.exec(source.slice(pos))[0]
+        tokens.push({
+          type: tokenTypes.LITERAL,
+          line: line,
+          value: value.trim(),
+        })
+        pos += value.length
+      }
     }
-  }
 
-  tokens.push({
-    type: tokenTypes.EOF,
-    line: line,
-  })
-  return tokens
-}
-
-module.exports = {
-  scan: scan,
-  tokenTypes: tokenTypes,
+    tokens.push({
+      type: tokenTypes.EOF,
+      line: line,
+    })
+    return tokens
+  }
 }

src/tokenStream.js

@@ -0,0 +1,8 @@
+module.exports = class TokenStream extends Array {
+  eat(tokenType) {
+    if (this[0] && this[0].type === tokenType) {
+      return this[0]
+    }
+    return false
+  }
+}

src/tokenTypes.js

@@ -0,0 +1,9 @@
+module.exports = {
+  OPAREN: '(',
+  CPAREN: ')',
+  KEYWORD: 'keyword',
+  QUOTE: '"',
+  LITERAL: 'literal',
+  ATTRIBUTE: 'attribute',
+  EOF: 'eof',
+}

test/lexer.js

@@ -1,10 +1,11 @@
 const test = require('tape')
-const lexer = require('../src/lexer')
 
-const tt = lexer.tokenTypes
+const Lexer = require('../src/lexer')
+const tt = require('../src/tokenTypes')
 
 test('lexes simple template correctly', t => {
   t.plan(4)
+  const lexer = new Lexer()
   let tokens = lexer.scan(
     '(div :class "foobar" (p "Lorem ipsum dolor sit amet"))',
   )
@@ -31,6 +32,7 @@ test('lexes simple template correctly', t => {
 
 test('keeps track of line numbers', t => {
   t.plan(2)
+  const lexer = new Lexer()
   let tokens = lexer.scan(`(
     (div :class "foobar"
       (p :class "bazquux"))