Lexer

Cargo.lock

@@ -0,0 +1,47 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "aho-corasick"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "elastic"
+version = "0.1.0"
+dependencies = [
+ "lazy_static",
+ "regex",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "memchr"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
+
+[[package]]
+name = "regex"
+version = "1.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"

Cargo.toml

@@ -7,3 +7,5 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+lazy_static = "1.4.0"
+regex = "1.4.5"

src/lexer.rs

@@ -0,0 +1,116 @@
+use lazy_static::lazy_static;
+use regex::Regex;
+
+#[derive(Debug, PartialEq)]
+pub struct LexerError {
+    message: String,
+}
+
+#[derive(Debug, PartialEq)]
+pub struct Token {
+    token_type: TokenType,
+    value: String,
+}
+
+impl Token {
+    pub fn new(token_type: TokenType, value: &str) -> Self {
+        Token {
+            token_type,
+            value: value.to_string(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum TokenType {
+    Identfiier,
+    Keyword,
+    Number,
+    String,
+}
+
+#[derive(Debug)]
+struct Matcher {
+    regex: Regex,
+    token_type: TokenType,
+}
+
+impl Matcher {
+    pub fn new(regex: &str, token_type: TokenType) -> Self {
+        Self {
+            regex: Regex::new(regex).unwrap(),
+            token_type,
+        }
+    }
+}
+
+lazy_static! {
+    static ref MATCHERS: Vec<Matcher> = vec![
+        Matcher::new(r#""(.*)""#, TokenType::String),
+        Matcher::new(r#"SELECT"#, TokenType::Keyword),
+        Matcher::new(r#"[a-z][a-zA-Z_]*"#, TokenType::Identfiier),
+        Matcher::new(r#"[0-9]+"#, TokenType::Number)
+    ];
+}
+
+pub fn scan(input: &str) -> Result<Vec<Token>, LexerError> {
+    let mut tokens: Vec<Token> = vec![];
+    let mut position = 0;
+
+    while position < input.len() {
+        for matcher in MATCHERS.iter() {
+            if matcher.regex.is_match(&input[position..]) {
+                dbg!(&input[position..]);
+                if let Some(m) = matcher.regex.captures_iter(&input[position..]).next() {
+                    let value = if m.len() > 1 { &m[1] } else { &m[0] };
+                    position += value.len();
+                    if matcher.token_type == TokenType::String {
+                        position += 2
+                    };
+                    tokens.push(Token::new(matcher.token_type, value));
+                }
+
+                break;
+            }
+        }
+    }
+
+    Ok(tokens)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_scans_a_number() {
+        assert_eq!(
+            scan("123").unwrap(),
+            vec![Token::new(TokenType::Number, "123")]
+        )
+    }
+
+    #[test]
+    fn it_scans_a_string() {
+        assert_eq!(
+            scan("\"hello world\"").unwrap(),
+            vec![Token::new(TokenType::String, "hello world")]
+        )
+    }
+
+    #[test]
+    fn it_scans_a_keyword() {
+        assert_eq!(
+            scan("SELECT").unwrap(),
+            vec![Token::new(TokenType::Keyword, "SELECT")]
+        )
+    }
+
+    #[test]
+    fn it_scans_an_identifier() {
+        assert_eq!(
+            scan("abc").unwrap(),
+            vec![Token::new(TokenType::Identfiier, "abc")]
+        )
+    }
+}

src/lib.rs

@@ -0,0 +1 @@
+mod lexer;