diff options
author | Aleksey Kladov <[email protected]> | 2018-08-18 10:13:34 +0100 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2018-08-18 10:13:34 +0100 |
commit | a5eeef0eeed092cb663afc3b0cda2c0df0c7e793 (patch) | |
tree | 21b328cfe466870edf586ca58a115e1171b2f54f /crates/libsyntax2 | |
parent | c7b1be6be345f97d6c4fd9ff3c51a94fb817fa56 (diff) |
better char lexing
Diffstat (limited to 'crates/libsyntax2')
-rw-r--r-- | crates/libsyntax2/src/lexer/strings.rs | 28 | ||||
-rw-r--r-- | crates/libsyntax2/tests/data/lexer/0006_chars.rs | 2 | ||||
-rw-r--r-- | crates/libsyntax2/tests/data/lexer/0006_chars.txt | 10 |
3 files changed, 27 insertions, 13 deletions
diff --git a/crates/libsyntax2/src/lexer/strings.rs b/crates/libsyntax2/src/lexer/strings.rs index fbae767e5..795ea97b7 100644 --- a/crates/libsyntax2/src/lexer/strings.rs +++ b/crates/libsyntax2/src/lexer/strings.rs | |||
@@ -15,14 +15,23 @@ pub(crate) fn is_string_literal_start(c: char, c1: Option<char>, c2: Option<char | |||
15 | } | 15 | } |
16 | 16 | ||
17 | pub(crate) fn scan_char(ptr: &mut Ptr) { | 17 | pub(crate) fn scan_char(ptr: &mut Ptr) { |
18 | if ptr.bump().is_none() { | 18 | loop { |
19 | return; // TODO: error reporting is upper in the stack | 19 | if ptr.next_is('\\') { |
20 | } | 20 | ptr.bump(); |
21 | scan_char_or_byte(ptr); | 21 | if ptr.next_is('\\') || ptr.next_is('\'') { |
22 | if !ptr.next_is('\'') { | 22 | ptr.bump(); |
23 | return; // TODO: error reporting | 23 | } |
24 | continue; | ||
25 | } | ||
26 | if ptr.next_is('\'') { | ||
27 | ptr.bump(); | ||
28 | return; | ||
29 | } | ||
30 | if ptr.next_is('\n') { | ||
31 | break; | ||
32 | } | ||
33 | ptr.bump(); | ||
24 | } | 34 | } |
25 | ptr.bump(); | ||
26 | } | 35 | } |
27 | 36 | ||
28 | pub(crate) fn scan_byte_char_or_string(ptr: &mut Ptr) -> SyntaxKind { | 37 | pub(crate) fn scan_byte_char_or_string(ptr: &mut Ptr) -> SyntaxKind { |
@@ -111,8 +120,3 @@ fn scan_raw_byte_string(ptr: &mut Ptr) { | |||
111 | } | 120 | } |
112 | } | 121 | } |
113 | } | 122 | } |
114 | |||
115 | fn scan_char_or_byte(ptr: &mut Ptr) { | ||
116 | //FIXME: deal with escape sequencies | ||
117 | ptr.bump(); | ||
118 | } | ||
diff --git a/crates/libsyntax2/tests/data/lexer/0006_chars.rs b/crates/libsyntax2/tests/data/lexer/0006_chars.rs index 03598d908..454ee0a5f 100644 --- a/crates/libsyntax2/tests/data/lexer/0006_chars.rs +++ b/crates/libsyntax2/tests/data/lexer/0006_chars.rs | |||
@@ -1 +1 @@ | |||
'x' ' ' '0' | 'x' ' ' '0' 'hello' '\x7f' '\n' '\\' '\'' | ||
diff --git a/crates/libsyntax2/tests/data/lexer/0006_chars.txt b/crates/libsyntax2/tests/data/lexer/0006_chars.txt index ecaf22355..950954fbc 100644 --- a/crates/libsyntax2/tests/data/lexer/0006_chars.txt +++ b/crates/libsyntax2/tests/data/lexer/0006_chars.txt | |||
@@ -3,4 +3,14 @@ WHITESPACE 1 " " | |||
3 | CHAR 3 "\' \'" | 3 | CHAR 3 "\' \'" |
4 | WHITESPACE 1 " " | 4 | WHITESPACE 1 " " |
5 | CHAR 3 "\'0\'" | 5 | CHAR 3 "\'0\'" |
6 | WHITESPACE 1 " " | ||
7 | CHAR 7 "\'hello\'" | ||
8 | WHITESPACE 1 " " | ||
9 | CHAR 6 "\'\\x7f\'" | ||
10 | WHITESPACE 1 " " | ||
11 | CHAR 4 "\'\\n\'" | ||
12 | WHITESPACE 1 " " | ||
13 | CHAR 4 "\'\\\\\'" | ||
14 | WHITESPACE 1 " " | ||
15 | CHAR 4 "\'\\\'\'" | ||
6 | WHITESPACE 1 "\n" | 16 | WHITESPACE 1 "\n" |