123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111 |
- const TokenStream = require("./tokenStream");
- const tokenTypes = require("./tokenTypes");
-
- module.exports = class Lexer {
- scan(source) {
- let pos = 0;
- let line = 1;
- let tokenStream = new TokenStream();
- let allowSpecialCharactersInLiterals = false;
-
- while (pos < source.length) {
- if (source[pos].match(/\(/) && !allowSpecialCharactersInLiterals) {
- tokenStream.tokens.push({
- type: tokenTypes.OPAREN,
- line: line
- });
- pos++;
- } else if (source[pos].match(/\)/)) {
- tokenStream.tokens.push({
- type: tokenTypes.CPAREN,
- line: line
- });
- pos++;
- } else if (source[pos].match(/["]/)) {
- allowSpecialCharactersInLiterals = !allowSpecialCharactersInLiterals;
- tokenStream.tokens.push({
- type: tokenTypes.QUOTE,
- line: line
- });
- pos++;
- } else if (source[pos].match(/:/)) {
- let value = /:([^()'"\s]+)/.exec(source.slice(pos))[1].trim();
- tokenStream.tokens.push({
- type: tokenTypes.ATTRIBUTE,
- line: line,
- value: value
- });
- pos += value.length + 1; // the +1 is to account for the colon
- } else if (source[pos].match(/\'/)) {
- let value = /'([^()"\s]+)/.exec(source.slice(pos))[1].trim();
- tokenStream.tokens.push({
- type: tokenTypes.SYMBOL,
- line: line,
- value: value
- });
- pos += value.length + 1; // the +1 is to account for the apostrophe
- } else if (source[pos].match(/\,/)) {
- tokenStream.tokens.push({
- type: tokenTypes.COMMA,
- line: line,
- value: value
- });
- pos += 1;
- } else if (source[pos].match(/\d/)) {
- let number = "";
- while (source[pos] && source[pos].match(/\d/)) {
- number += source[pos];
- pos++;
- }
- tokenStream.tokens.push({
- type: tokenTypes.NUMBER,
- line: line,
- value: parseFloat(number)
- });
- } else if (source.slice(pos).match(/^#(t|f)/)) {
- pos++;
- tokenStream.tokens.push({
- type: tokenTypes.BOOLEAN,
- line: line,
- value: source[pos] === "t" ? true : false
- });
- pos++;
- } else if (source[pos].match(/\n/)) {
- line++;
- pos++;
- } else if (source[pos].match(/\s/)) {
- pos++;
- } else {
- let endPattern = /[^()"':\s]+/;
-
- if (allowSpecialCharactersInLiterals) {
- endPattern = /[^"']+/;
- }
-
- let value = endPattern.exec(source.slice(pos))[0].trim();
-
- if (allowSpecialCharactersInLiterals) {
- tokenStream.tokens.push({
- type: tokenTypes.LITERAL,
- line: line,
- value: value
- });
- } else {
- tokenStream.tokens.push({
- type: tokenTypes.IDENTIFIER,
- line: line,
- value: value.trim()
- });
- }
-
- pos += value.length;
- }
- }
-
- tokenStream.tokens.push({
- type: tokenTypes.EOF,
- line: line
- });
- return tokenStream;
- }
- };
|