Fix lexer whitespace issues

lib/chervil/lexer.rb

@@ -14,6 +14,10 @@ module Chervil
     end
 
     def get_next_token
+      while @source.slice(@position..-1).match(/^\s/)
+        advance
+      end
+
       case current_char
       when nil
         Token.new(:eof, "eof")
@@ -41,17 +45,14 @@ module Chervil
         Token.new(:string, string)
       else
         source = @source.slice(@position..-1)
-        while source.match(/^\s/)
-          advance
-          source = @source.slice(@position..-1)
-        end
-
         if match = source.match(/^[0-9]+(\.[0-9]+)?/)
           advance(match[0].size)
           Token.new(:number, match[0])
-        elsif match = source.match(/^[a-z!$%&*\/:<=>?~_^+][a-z0-9@!$%&*\/:<=>?~_^,\-]*/)
+        elsif match = source.match(/^[a-z!$%&*\/:<=>?~_^+\-][a-z0-9@!$%&*\/:<=>?~_^+\-]*/)
           advance(match[0].size)
           Token.new(:identifier, match[0])
+        else
+          raise "Unrecognized character #{current_char}"
         end
       end
     end
@@ -60,7 +61,7 @@ module Chervil
       tokens = Array.new
       loop do
         token = get_next_token
-        tokens << token
+        tokens << token unless token.nil?
         break if token.type == :eof
       end
       tokens