diff options
-rw-r--r-- | src/lexer/mod.rs | 8 | ||||
-rw-r--r-- | tests/data/lexer/0003_ident.rs | 1 | ||||
-rw-r--r-- | tests/data/lexer/0003_ident.txt | 14 | ||||
-rw-r--r-- | validation.md | 1 |
4 files changed, 24 insertions, 0 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index ecea664da..83a411cdd 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs | |||
@@ -22,6 +22,14 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
22 | // They are not identifiers, and are handled further down. | 22 | // They are not identifiers, and are handled further down. |
23 | let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext()); | 23 | let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext()); |
24 | if ident_start { | 24 | if ident_start { |
25 | let is_single_letter = match ptr.next() { | ||
26 | None => true, | ||
27 | Some(c) if !is_ident_continue(c) => true, | ||
28 | _ => false, | ||
29 | }; | ||
30 | if is_single_letter { | ||
31 | return if c == '_' { UNDERSCORE } else { IDENT }; | ||
32 | } | ||
25 | ptr.bump_while(is_ident_continue); | 33 | ptr.bump_while(is_ident_continue); |
26 | return IDENT; | 34 | return IDENT; |
27 | } | 35 | } |
diff --git a/tests/data/lexer/0003_ident.rs b/tests/data/lexer/0003_ident.rs new file mode 100644 index 000000000..c05c9c009 --- /dev/null +++ b/tests/data/lexer/0003_ident.rs | |||
@@ -0,0 +1 @@ | |||
foo foo_ _foo _ __ x привет | |||
diff --git a/tests/data/lexer/0003_ident.txt b/tests/data/lexer/0003_ident.txt new file mode 100644 index 000000000..eec82fb91 --- /dev/null +++ b/tests/data/lexer/0003_ident.txt | |||
@@ -0,0 +1,14 @@ | |||
1 | IDENT 3 | ||
2 | WHITESPACE 1 | ||
3 | IDENT 4 | ||
4 | WHITESPACE 1 | ||
5 | IDENT 4 | ||
6 | WHITESPACE 1 | ||
7 | UNDERSCORE 1 | ||
8 | WHITESPACE 1 | ||
9 | IDENT 2 | ||
10 | WHITESPACE 1 | ||
11 | IDENT 1 | ||
12 | WHITESPACE 1 | ||
13 | IDENT 12 | ||
14 | WHITESPACE 1 | ||
diff --git a/validation.md b/validation.md index 9cfec5309..3706760ba 100644 --- a/validation.md +++ b/validation.md | |||
@@ -1,4 +1,5 @@ | |||
1 | Fixmes: | 1 | Fixmes: |
2 | 2 | ||
3 | * Fix `is_whitespace`, add more test | 3 | * Fix `is_whitespace`, add more test |
4 | * Add more thorough tests for idents for XID_Start & XID_Continue | ||
4 | 5 | ||