|
@@ -14,6 +14,10 @@ module Chervil
|
14
|
14
|
end
|
15
|
15
|
|
16
|
16
|
def get_next_token
|
|
17
|
+ while @source.slice(@position..-1).match(/^\s/)
|
|
18
|
+ advance
|
|
19
|
+ end
|
|
20
|
+
|
17
|
21
|
case current_char
|
18
|
22
|
when nil
|
19
|
23
|
Token.new(:eof, "eof")
|
|
@@ -41,17 +45,14 @@ module Chervil
|
41
|
45
|
Token.new(:string, string)
|
42
|
46
|
else
|
43
|
47
|
source = @source.slice(@position..-1)
|
44
|
|
- while source.match(/^\s/)
|
45
|
|
- advance
|
46
|
|
- source = @source.slice(@position..-1)
|
47
|
|
- end
|
48
|
|
-
|
49
|
48
|
if match = source.match(/^[0-9]+(\.[0-9]+)?/)
|
50
|
49
|
advance(match[0].size)
|
51
|
50
|
Token.new(:number, match[0])
|
52
|
|
- elsif match = source.match(/^[a-z!$%&*\/:<=>?~_^+][a-z0-9@!$%&*\/:<=>?~_^,\-]*/)
|
|
51
|
+ elsif match = source.match(/^[a-z!$%&*\/:<=>?~_^+\-][a-z0-9@!$%&*\/:<=>?~_^+\-]*/)
|
53
|
52
|
advance(match[0].size)
|
54
|
53
|
Token.new(:identifier, match[0])
|
|
54
|
+ else
|
|
55
|
+ raise "Unrecognized character #{current_char}"
|
55
|
56
|
end
|
56
|
57
|
end
|
57
|
58
|
end
|
|
@@ -60,7 +61,7 @@ module Chervil
|
60
|
61
|
tokens = Array.new
|
61
|
62
|
loop do
|
62
|
63
|
token = get_next_token
|
63
|
|
- tokens << token
|
|
64
|
+ tokens << token unless token.nil?
|
64
|
65
|
break if token.type == :eof
|
65
|
66
|
end
|
66
|
67
|
tokens
|