Allow using special characters in literals

src/lexer.js

@@ -6,10 +6,10 @@ module.exports = class Lexer {
     let pos = 0
     let line = 1
     let tokenStream = new TokenStream()
-    let allowWhiteSpaceInLiterals = false
+    let allowSpecialCharactersInLiterals = false
 
     while (pos < source.length) {
-      if (source[pos].match(/\(/)) {
+      if (source[pos].match(/\(/) && !allowSpecialCharactersInLiterals) {
         tokenStream.tokens.push({
           type: tokenTypes.OPAREN,
           line: line,
@@ -22,7 +22,7 @@ module.exports = class Lexer {
         })
         pos++
       } else if (source[pos].match(/['"]/)) {
-        allowWhiteSpaceInLiterals = !allowWhiteSpaceInLiterals
+        allowSpecialCharactersInLiterals = !allowSpecialCharactersInLiterals
         tokenStream.tokens.push({
           type: tokenTypes.QUOTE,
           line: line,
@@ -44,8 +44,8 @@ module.exports = class Lexer {
       } else {
         let endPattern = /[^()"':\s]+/
 
-        if (allowWhiteSpaceInLiterals) {
-          endPattern = /[^()"':]+/
+        if (allowSpecialCharactersInLiterals) {
+          endPattern = /[^"']+/
         }
 
         let value = endPattern.exec(source.slice(pos))[0]

test/lexer.js

@@ -72,3 +72,24 @@ test('multiple identifiers in a row are kept separate', t => {
     ]
   )
 })
+
+test('allow special characters inside quotes', t => {
+  t.plan(2)
+  const lexer = new Lexer()
+  let tokens = lexer.scan(`
+    (p "(test)")
+  `).tokens
+  t.deepEqual(
+    tokens.map(token => token.type),
+    [
+      tt.OPAREN,
+      tt.LITERAL,
+      tt.QUOTE,
+      tt.LITERAL,
+      tt.QUOTE,
+      tt.CPAREN,
+      tt.EOF,
+    ]
+  )
+  t.equal(tokens[3].value, '(test)')
+})