Browse Source

Give keywords their own token types

main
Dylan Baker 3 years ago
parent
commit
12db8c5658
2 changed files with 10 additions and 6 deletions
  1. 5
    4
      src/lexer.rs
  2. 5
    2
      src/token.rs

+ 5
- 4
src/lexer.rs View File

23
     static ref WHITESPACE_REGEX: Regex = Regex::new(r"\s").unwrap();
23
     static ref WHITESPACE_REGEX: Regex = Regex::new(r"\s").unwrap();
24
     static ref MATCHERS: Vec<Matcher> = vec![
24
     static ref MATCHERS: Vec<Matcher> = vec![
25
         Matcher::new(r#""(.*)""#, TokenType::String),
25
         Matcher::new(r#""(.*)""#, TokenType::String),
26
-        Matcher::new(r#"(SELECT|FROM)"#, TokenType::Keyword),
26
+        Matcher::new(r#"SELECT"#, TokenType::Select),
27
+        Matcher::new(r#"FROM"#, TokenType::From),
27
         Matcher::new(r#"[a-z][a-zA-Z_\-]*"#, TokenType::Identfiier),
28
         Matcher::new(r#"[a-z][a-zA-Z_\-]*"#, TokenType::Identfiier),
28
         Matcher::new(r#"[0-9]+"#, TokenType::Number),
29
         Matcher::new(r#"[0-9]+"#, TokenType::Number),
29
         Matcher::new(r#"\*"#, TokenType::Star),
30
         Matcher::new(r#"\*"#, TokenType::Star),
82
     fn it_scans_a_keyword() {
83
     fn it_scans_a_keyword() {
83
         assert_eq!(
84
         assert_eq!(
84
             scan("SELECT").unwrap(),
85
             scan("SELECT").unwrap(),
85
-            vec![Token::new(TokenType::Keyword, "SELECT")]
86
+            vec![Token::new(TokenType::Select, "SELECT")]
86
         )
87
         )
87
     }
88
     }
88
 
89
 
91
         assert_eq!(
92
         assert_eq!(
92
             scan("SELECT FROM").unwrap(),
93
             scan("SELECT FROM").unwrap(),
93
             vec![
94
             vec![
94
-                Token::new(TokenType::Keyword, "SELECT"),
95
-                Token::new(TokenType::Keyword, "FROM")
95
+                Token::new(TokenType::Select, "SELECT"),
96
+                Token::new(TokenType::From, "FROM")
96
             ]
97
             ]
97
         )
98
         )
98
     }
99
     }

+ 5
- 2
src/token.rs View File

25
 
25
 
26
 #[derive(Clone, Copy, Debug, PartialEq)]
26
 #[derive(Clone, Copy, Debug, PartialEq)]
27
 pub enum TokenType {
27
 pub enum TokenType {
28
+    From,
28
     Identfiier,
29
     Identfiier,
29
-    Keyword,
30
     Number,
30
     Number,
31
+    Select,
31
     Star,
32
     Star,
32
     String,
33
     String,
33
 }
34
 }
39
             "{}",
40
             "{}",
40
             match self {
41
             match self {
41
                 TokenType::Identfiier => "Identifier",
42
                 TokenType::Identfiier => "Identifier",
42
-                TokenType::Keyword => "Keyword",
43
                 TokenType::Number => "Number",
43
                 TokenType::Number => "Number",
44
                 TokenType::Star => "Star",
44
                 TokenType::Star => "Star",
45
                 TokenType::String => "String",
45
                 TokenType::String => "String",
46
+
47
+                TokenType::Select => "SELECT",
48
+                TokenType::From => "FROM",
46
             }
49
             }
47
         )
50
         )
48
     }
51
     }

Loading…
Cancel
Save