sexp-template/src/lexer.js

const TokenStream = require("./tokenStream");
const tokenTypes = require("./tokenTypes");

module.exports = class Lexer {
  scan(source) {
    let pos = 0;
    let line = 1;
    let tokenStream = new TokenStream();
    let allowSpecialCharactersInLiterals = false;

    while (pos < source.length) {
      if (source[pos].match(/\(/) && !allowSpecialCharactersInLiterals) {
        tokenStream.tokens.push({
          type: tokenTypes.OPAREN,
          line: line
        });
        pos++;
      } else if (source[pos].match(/\)/)) {
        tokenStream.tokens.push({
          type: tokenTypes.CPAREN,
          line: line
        });
        pos++;
      } else if (source[pos].match(/["]/)) {
        allowSpecialCharactersInLiterals = !allowSpecialCharactersInLiterals;
        tokenStream.tokens.push({
          type: tokenTypes.QUOTE,
          line: line
        });
        pos++;
      } else if (source[pos].match(/:/)) {
        let value = /:([^()'"\s]+)/.exec(source.slice(pos))[1].trim();
        tokenStream.tokens.push({
          type: tokenTypes.ATTRIBUTE,
          line: line,
          value: value
        });
        pos += value.length + 1; // the +1 is to account for the colon
      } else if (source[pos].match(/\'/)) {
        let value = /'([^()"\s]+)/.exec(source.slice(pos))[1].trim();
        tokenStream.tokens.push({
          type: tokenTypes.SYMBOL,
          line: line,
          value: value
        });
        pos += value.length + 1; // the +1 is to account for the apostrophe
      } else if (source[pos].match(/\d/)) {
        let number = "";
        while (source[pos] && source[pos].match(/\d/)) {
          number += source[pos];
          pos++;
        }
        tokenStream.tokens.push({
          type: tokenTypes.NUMBER,
          line: line,
          value: parseFloat(number)
        });
      } else if (source.slice(pos).match(/^#(t|f)/)) {
        pos++;
        tokenStream.tokens.push({
          type: tokenTypes.BOOLEAN,
          line: line,
          value: source[pos] === "t" ? true : false
        });
        pos++;
      } else if (source[pos].match(/\n/)) {
        line++;
        pos++;
      } else if (source[pos].match(/\s/)) {
        pos++;
      } else {
        let endPattern = /[^()"':\s]+/;

        if (allowSpecialCharactersInLiterals) {
          endPattern = /[^"']+/;
        }

        let value = endPattern.exec(source.slice(pos))[0].trim();

        if (allowSpecialCharactersInLiterals) {
          tokenStream.tokens.push({
            type: tokenTypes.LITERAL,
            line: line,
            value: value
          });
        } else {
          tokenStream.tokens.push({
            type: tokenTypes.IDENTIFIER,
            line: line,
            value: value.trim()
          });
        }

        pos += value.length;
      }
    }

    tokenStream.tokens.push({
      type: tokenTypes.EOF,
      line: line
    });
    return tokenStream;
  }
};