sexp-template/src/lexer.js

const TokenStream = require('./tokenStream')
const tokenTypes = require('./tokenTypes')

module.exports = class Lexer {
  scan(source) {
    let pos = 0
    let line = 1
    let tokenStream = new TokenStream()
    let allowSpecialCharactersInLiterals = false

    while (pos < source.length) {
      if (source[pos].match(/\(/) && !allowSpecialCharactersInLiterals) {
        tokenStream.tokens.push({
          type: tokenTypes.OPAREN,
          line: line,
        })
        pos++
      } else if (source[pos].match(/\)/)) {
        tokenStream.tokens.push({
          type: tokenTypes.CPAREN,
          line: line,
        })
        pos++
      } else if (source[pos].match(/["]/)) {
        allowSpecialCharactersInLiterals = !allowSpecialCharactersInLiterals
        tokenStream.tokens.push({
          type: tokenTypes.QUOTE,
          line: line,
        })
        pos++
      } else if (source[pos].match(/:/)) {
        let value = /:([^()'"\s]+)/.exec(source.slice(pos))[1].trim()
        tokenStream.tokens.push({
          type: tokenTypes.ATTRIBUTE,
          line: line,
          value: value,
        })
        pos += value.length + 1 // the +1 is to account for the colon
      } else if (source[pos].match(/\'/)) {
        let value = /'([^()"\s]+)/.exec(source.slice(pos))[1].trim()
        tokenStream.tokens.push({
          type: tokenTypes.SYMBOL,
          line: line,
          value: value,
        })
        pos += value.length + 1 // the +1 is to account for the apostrophe
      } else if (source[pos].match(/\,/)) {
        tokenStream.tokens.push({
          type: tokenTypes.COMMA,
          line: line,
          value: value,
        })
        pos += 1
      } else if (source[pos].match(/\d/)) {
        let number = ''
        while (source[pos] && source[pos].match(/\d/)) {
          number += source[pos]
          pos++
        }
        tokenStream.tokens.push({
          type: tokenTypes.NUMBER,
          line: line,
          value: parseFloat(number),
        })
      } else if (source.slice(pos).match(/^#(t|f)/)) {
        pos++
        tokenStream.tokens.push({
          type: tokenTypes.BOOLEAN,
          line: line,
          value: source[pos] === 't' ? true : false,
        })
        pos++
      } else if (source[pos].match(/\n/)) {
        line++
        pos++
      } else if (source[pos].match(/\s/)) {
        pos++
      } else {
        let endPattern = /[^()"':\s]+/

        if (allowSpecialCharactersInLiterals) {
          endPattern = /[^"']+/
        }

        let value = endPattern.exec(source.slice(pos))[0].trim()

        if (allowSpecialCharactersInLiterals) {
          tokenStream.tokens.push({
            type: tokenTypes.LITERAL,
            line: line,
            value: value,
          })
        } else {
          tokenStream.tokens.push({
            type: tokenTypes.IDENTIFIER,
            line: line,
            value: value.trim(),
          })
        }

        pos += value.length
      }
    }

    tokenStream.tokens.push({
      type: tokenTypes.EOF,
      line: line,
    })
    return tokenStream
  }
}