Initial commit/lexer

.gitignore

@@ -0,0 +1,2 @@
+node_modules
+src/index.ts

.mocharc.json

@@ -0,0 +1,5 @@
+{
+  "extension": ["ts"],
+  "spec": "test/*.test.ts",
+  "require": "ts-node/register"
+}

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "sql",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha",
+    "format": "tslint --fix ./{src,test}/*.ts"
+  },
+  "keywords": [],
+  "author": "Dylan Baker <dylan@simulacrum.party>",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/chai": "^4.1.7",
+    "@types/mocha": "^5.2.7",
+    "@types/node": "^12.6.9",
+    "chai": "^4.2.0",
+    "mocha": "^6.2.0",
+    "ts-node": "^8.3.0",
+    "tslint": "^5.18.0",
+    "typescript": "^3.5.3"
+  }
+}

src/error.ts

@@ -0,0 +1,13 @@
+export default class Error {
+  public message: string;
+  public line: number;
+
+  constructor(message: string, line: number) {
+    this.message = message;
+    this.line = line;
+  }
+}
+
+export const isError = (obj: any): obj is Error => {
+  return obj && obj.message && obj.line;
+};

src/lexer.ts

@@ -0,0 +1,105 @@
+import Error, { isError } from "./error";
+import Token, { TokenKind } from "./token";
+
+export default class Lexer {
+  public source: string;
+  public position: number;
+  public line: number;
+
+  constructor(source: string) {
+    this.source = source;
+    this.position = 0;
+    this.line = 1;
+  }
+  public scan(): Token[] | Error {
+    const tokens = [];
+
+    while (!this.atEnd()) {
+      const result = this.getToken();
+      if (isError(result)) {
+        return result;
+      } else if (result) {
+        tokens.push(result);
+      }
+    }
+
+    tokens.push(new Token(TokenKind.EOF, null, this.line));
+
+    return tokens;
+  }
+
+  private getToken(): Token | Error | null {
+    const source = this.source.slice(this.position);
+
+    if (source.match(/^select/i)) {
+      this.advance(6);
+      return new Token(TokenKind.SELECT, null, this.line);
+    } else if (source.match(/^where/i)) {
+      this.advance(5);
+      return new Token(TokenKind.WHERE, null, this.line);
+    } else if (source.match(/^from/i)) {
+      this.advance(4);
+      return new Token(TokenKind.FROM, null, this.line);
+    } else if (source.match(/^\*/)) {
+      this.advance();
+      return new Token(TokenKind.STAR, null, this.line);
+    } else if (source.match(/^=/)) {
+      this.advance();
+      return new Token(TokenKind.EQUALS, null, this.line);
+    } else if (source.match(/^,/)) {
+      this.advance();
+      return new Token(TokenKind.COMMA, null, this.line);
+    } else if (source.match(/^`/)) {
+      this.advance();
+      return new Token(TokenKind.BACKTICK, null, this.line);
+    } else if (source.match(/^\.([^0-9]|$)/)) {
+      this.advance();
+      return new Token(TokenKind.DOT, null, this.line);
+    } else if (source.match(/^;/)) {
+      this.advance();
+      return new Token(TokenKind.SEMICOLON, null, this.line);
+    } else if (source.match(/^[0-9]+(\.[0-9]+)?/)) {
+      const match = source.match(/^[0-9]+(\.[0-9]+)?/);
+      if (match) {
+        const numeric = match[0];
+        this.advance(numeric.length);
+        return new Token(TokenKind.NUMBER, numeric, this.line);
+      }
+    } else if (source.match(/^\.[0-9]+/)) {
+      const match = source.match(/^\.[0-9]+/);
+      if (match) {
+        const numeric = match[0];
+        this.advance(numeric.length);
+        return new Token(TokenKind.NUMBER, numeric, this.line);
+      }
+    } else if (source.match(/^[a-zA-Z_][a-zA-Z0-9_]*/)) {
+      const match = source.match(/^[a-zA-Z_][a-zA-Z0-9_]*/);
+      if (match) {
+        const identifier = match[0];
+        this.advance(identifier.length);
+        return new Token(TokenKind.IDENTIFIER, identifier, this.line);
+      }
+    } else if (source.match(/^\n/)) {
+      this.advance();
+      this.nextLine();
+      return null;
+    } else if (source.match(/^\s/)) {
+      this.advance();
+      return null;
+    }
+
+    return new Error(`Unrecognized character ${source[0]}`, this.line);
+  }
+
+  private advance(step: number = 1) {
+    this.position += step;
+  }
+
+  private nextLine() {
+    this.line += 1;
+  }
+
+  private atEnd(): boolean {
+    return this.position === this.source.length;
+  }
+}

src/token.ts

@@ -0,0 +1,26 @@
+export enum TokenKind {
+  BACKTICK = "BACKTICK",
+  COMMA = "COMMA",
+  DOT = "DOT",
+  EOF = "EOF",
+  EQUALS = "EQUALS",
+  FROM = "FROM",
+  IDENTIFIER = "IDENTIFIER",
+  NUMBER = "NUMBER",
+  SELECT = "SELECT",
+  SEMICOLON = "SEMICOLON",
+  STAR = "STAR",
+  WHERE = "WHERE",
+}
+
+export default class Token {
+  public kind: TokenKind;
+  public value: string | null;
+  public line: number;
+
+  constructor(kind: TokenKind, value: string | null, line: number) {
+    this.kind = kind;
+    this.value = value;
+    this.line = line;
+  }
+}

test/lexer.test.ts

@@ -0,0 +1,82 @@
+/* tslint:disable:no-unused-expression */
+import { expect } from "chai";
+
+import Error, { isError } from "../src/error";
+import Lexer from "../src/lexer";
+import Token, { TokenKind } from "../src/token";
+
+const scan = (source: string): Token[] | Error => {
+  return new Lexer(source).scan();
+};
+
+describe("Lexer", () => {
+  it("scans uppercase keywords", () => {
+    const tokens = scan("SELECT FROM WHERE");
+    expect(tokens).to.deep.equal([
+      new Token(TokenKind.SELECT, null, 1),
+      new Token(TokenKind.FROM, null, 1),
+      new Token(TokenKind.WHERE, null, 1),
+      new Token(TokenKind.EOF, null, 1),
+    ]);
+  });
+
+  it("scans lowercase keywords", () => {
+    const tokens = scan("select from where");
+    expect(tokens).to.deep.equal([
+      new Token(TokenKind.SELECT, null, 1),
+      new Token(TokenKind.FROM, null, 1),
+      new Token(TokenKind.WHERE, null, 1),
+      new Token(TokenKind.EOF, null, 1),
+    ]);
+  });
+
+  it("scans symbols", () => {
+    const tokens = scan("*=,`;.");
+    expect(tokens).to.deep.equal([
+      new Token(TokenKind.STAR, null, 1),
+      new Token(TokenKind.EQUALS, null, 1),
+      new Token(TokenKind.COMMA, null, 1),
+      new Token(TokenKind.BACKTICK, null, 1),
+      new Token(TokenKind.SEMICOLON, null, 1),
+      new Token(TokenKind.DOT, null, 1),
+      new Token(TokenKind.EOF, null, 1),
+    ]);
+  });
+
+  it("scans identifiers", () => {
+    const tokens = scan("ident ident_ifier Ident");
+    expect(tokens).to.deep.equal([
+      new Token(TokenKind.IDENTIFIER, "ident", 1),
+      new Token(TokenKind.IDENTIFIER, "ident_ifier", 1),
+      new Token(TokenKind.IDENTIFIER, "Ident", 1),
+      new Token(TokenKind.EOF, null, 1),
+    ]);
+  });
+
+  it("scans numbers", () => {
+    const tokens = scan("1 123.45 .123");
+    expect(tokens).to.deep.equal([
+      new Token(TokenKind.NUMBER, "1", 1),
+      new Token(TokenKind.NUMBER, "123.45", 1),
+      new Token(TokenKind.NUMBER, ".123", 1),
+      new Token(TokenKind.EOF, null, 1),
+    ]);
+  });
+
+  it("keeps track of line numbers", () => {
+    const tokens = scan("foo\nbar");
+    expect(tokens).to.deep.equal([
+      new Token(TokenKind.IDENTIFIER, "foo", 1),
+      new Token(TokenKind.IDENTIFIER, "bar", 2),
+      new Token(TokenKind.EOF, null, 2),
+    ]);
+  });
+
+  it("returns an error for an unrecognized character", () => {
+    const error = scan("~");
+    expect(isError(error)).to.be.ok;
+    if (isError(error)) {
+      expect(error.message).to.equal("Unrecognized character ~");
+    }
+  });
+});

tsconfig.json

@@ -0,0 +1,62 @@
+{
+  "compilerOptions": {
+    /* Basic Options */
+    "target": "es5",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    // "lib": [],                             /* Specify library files to be included in the compilation. */
+    // "allowJs": true,                       /* Allow javascript files to be compiled. */
+    // "checkJs": true,                       /* Report errors in .js files. */
+    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
+    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
+    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
+    // "outFile": "./",                       /* Concatenate and emit output to single file. */
+    // "outDir": "./",                        /* Redirect output structure to the directory. */
+    // "rootDir": "./",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    // "composite": true,                     /* Enable project compilation */
+    // "incremental": true,                   /* Enable incremental compilation */
+    // "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+    // "removeComments": true,                /* Do not emit comments to output. */
+    // "noEmit": true,                        /* Do not emit outputs. */
+    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+    /* Strict Type-Checking Options */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,              /* Enable strict null checks. */
+    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
+    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
+
+    /* Additional Checks */
+    // "noUnusedLocals": true,                /* Report errors on unused locals. */
+    // "noUnusedParameters": true,            /* Report errors on unused parameters. */
+    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+    /* Module Resolution Options */
+    // "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+    // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+    // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+    // "typeRoots": [],                       /* List of folders to include type definitions from. */
+    // "types": [],                           /* Type declaration files to be included in compilation. */
+    // "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+
+    /* Source Map Options */
+    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
+    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+    /* Experimental Options */
+    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+  }
+}

tslint.json

@@ -0,0 +1,18 @@
+{
+  "defaultSeverity": "error",
+  "extends": ["tslint:recommended"],
+  "jsRules": {},
+  "rules": {},
+  "rulesDirectory": [],
+  "linterOptions": {
+    "exclude": ["src/index.ts"]
+  },
+  "overrides": [
+    {
+      "files": ["*.test.ts"],
+      "rules": {
+        "no-unused-expressions": "off"
+      }
+    }
+  ]
+}