|
@@ -5,9 +5,9 @@ class Lexer
|
5
|
5
|
end
|
6
|
6
|
|
7
|
7
|
def get_token
|
8
|
|
- return Token.new(TokenKinds::EOF) if at_end
|
|
8
|
+ skip_whitespace
|
9
|
9
|
|
10
|
|
- @position += 1 while !at_end && @source.slice(@position).match(/\s/)
|
|
10
|
+ return Token.new(TokenKinds::EOF) if at_end
|
11
|
11
|
|
12
|
12
|
source = @source.slice(@position..-1)
|
13
|
13
|
|
|
@@ -176,18 +176,19 @@ class Lexer
|
176
|
176
|
def scan_all
|
177
|
177
|
tokens = Array.new
|
178
|
178
|
until at_end
|
179
|
|
- if @source.slice(@position).match(/\s/)
|
180
|
|
- @position += 1
|
181
|
|
- else
|
182
|
|
- tokens << get_token
|
183
|
|
- end
|
|
179
|
+ tokens << get_token
|
184
|
180
|
end
|
185
|
181
|
tokens << Token.new(TokenKinds::EOF)
|
186
|
|
- tokens
|
187
|
182
|
end
|
188
|
183
|
|
189
|
184
|
private
|
190
|
185
|
|
|
186
|
+ def skip_whitespace
|
|
187
|
+ while !at_end && @source[@position].match(/\A\s/)
|
|
188
|
+ @position += 1
|
|
189
|
+ end
|
|
190
|
+ end
|
|
191
|
+
|
191
|
192
|
def at_end
|
192
|
193
|
@position == @source.size
|
193
|
194
|
end
|