Browse Source

Escape quotes, newlines, tabs

master
Dylan Baker 4 years ago
parent
commit
956369475a
2 changed files with 48 additions and 3 deletions
  1. 29
    3
      lib/ahem/lexer.rb
  2. 19
    0
      spec/lexer_spec.rb

+ 29
- 3
lib/ahem/lexer.rb View File

@@ -32,9 +32,35 @@ class Lexer
32 32
       number = source.match(/^\d+(\.\d+)?/)[0]
33 33
       @position += number.size
34 34
       Token.new(TokenKinds::NUMBER, number.to_f)
35
-    elsif source.match(/^"(.*)"/)
36
-      string = source.match(/^"([^"]*)"/)[1]
37
-      @position += (string.size + 2)
35
+    elsif source[0] == '"'
36
+      @position += 1
37
+      string = String.new
38
+
39
+      while !at_end && @source[@position] != '"'
40
+        if @source[@position] == '\\'
41
+          if @source[@position + 1] == '"'
42
+            @position += 1
43
+          elsif @source[@position + 1] == 'n'
44
+            @position += 2
45
+            string << "\n"
46
+            next
47
+          elsif @source[@position + 1] == 't'
48
+            @position += 2
49
+            string << "\t"
50
+            next
51
+          end
52
+        end
53
+
54
+        string << @source[@position]
55
+        @position += 1
56
+      end
57
+
58
+      if at_end
59
+        raise 'Unterminated string'
60
+      elsif @source[@position] == '"'
61
+        @position += 1
62
+      end
63
+
38 64
       Token.new(TokenKinds::STRING, string)
39 65
     elsif source.match(/^\:([a-z][a-zA-Z0-9_]*)/)
40 66
       atom = source.match(/^\:([a-z][a-zA-Z0-9_]*)/)[1]

+ 19
- 0
spec/lexer_spec.rb View File

@@ -147,4 +147,23 @@ RSpec.describe Lexer do
147 147
       ]
148 148
     )
149 149
   end
150
+
151
+  it 'allows escaping quotes in strings' do
152
+    expect(Lexer.new('"hello \"world\""').scan_all).to eq(
153
+      [
154
+        Token.new(TokenKinds::STRING, 'hello "world"'),
155
+        Token.new(TokenKinds::EOF),
156
+      ]
157
+    )
158
+  end
159
+
160
+  it 'supports newlines and tabs in strings' do
161
+    expect(Lexer.new('"hello\nworld" "hello\tworld"').scan_all).to eq(
162
+      [
163
+        Token.new(TokenKinds::STRING, "hello\nworld"),
164
+        Token.new(TokenKinds::STRING, "hello\tworld"),
165
+        Token.new(TokenKinds::EOF),
166
+      ]
167
+    )
168
+  end
150 169
 end

Loading…
Cancel
Save