Allow whitespace while between quotes

What was happening was that spaces were allowed in
all literals to account for quoted strings, but
this meant that two identifiers in a row would get
parsed as a single literal. This commit sets a
flag to allow whitespace in literals when a quote
is encountered and then unsets that flag at the
next quote.

src/lexer.js

@@ -6,6 +6,7 @@ module.exports = class Lexer {
     let pos = 0
     let line = 1
     let tokenStream = new TokenStream()
+    let allowWhiteSpaceInLiterals = false
 
     while (pos < source.length) {
       if (source[pos].match(/\(/)) {
@@ -21,6 +22,7 @@ module.exports = class Lexer {
         })
         pos++
       } else if (source[pos].match(/['"]/)) {
+        allowWhiteSpaceInLiterals = !allowWhiteSpaceInLiterals
         tokenStream.tokens.push({
           type: tokenTypes.QUOTE,
           line: line,
@@ -40,7 +42,13 @@ module.exports = class Lexer {
       } else if (source[pos].match(/\s/)) {
         pos++
       } else {
-        let value = /[^()"':]+/.exec(source.slice(pos))[0]
+        let endPattern = /[^()"':\s]+/
+
+        if (allowWhiteSpaceInLiterals) {
+          endPattern = /[^()"':]+/
+        }
+
+        let value = endPattern.exec(source.slice(pos))[0]
         tokenStream.tokens.push({
           type: tokenTypes.LITERAL,
           line: line,

test/lexer.js

@@ -40,3 +40,35 @@ test('keeps track of line numbers', t => {
   t.equal(tokens[2].line, 2)
   t.equal(tokens[7].line, 3)
 })
+
+test('multiple identifiers in a row are kept separate', t => {
+  t.plan(2)
+  const lexer = new Lexer()
+  let tokens = lexer.scan(`(test test test)`).tokens
+  t.deepEqual(
+    tokens.map(token => token.type),
+    [
+      tt.OPAREN,
+      tt.LITERAL,
+      tt.LITERAL,
+      tt.LITERAL,
+      tt.CPAREN,
+      tt.EOF,
+    ]
+  )
+  tokens = lexer.scan(`(test "test" test test)`).tokens
+  t.deepEqual(
+    tokens.map(token => token.type),
+    [
+      tt.OPAREN,
+      tt.LITERAL,
+      tt.QUOTE,
+      tt.LITERAL,
+      tt.QUOTE,
+      tt.LITERAL,
+      tt.LITERAL,
+      tt.CPAREN,
+      tt.EOF,
+    ]
+  )
+})