aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/lexer/mod.rs8
-rw-r--r--tests/data/lexer/0003_ident.rs1
-rw-r--r--tests/data/lexer/0003_ident.txt14
-rw-r--r--validation.md1
4 files changed, 24 insertions, 0 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index ecea664da..83a411cdd 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -22,6 +22,14 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
22 // They are not identifiers, and are handled further down. 22 // They are not identifiers, and are handled further down.
23 let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext()); 23 let ident_start = is_ident_start(c) && !string_literal_start(c, ptr.next(), ptr.nnext());
24 if ident_start { 24 if ident_start {
25 let is_single_letter = match ptr.next() {
26 None => true,
27 Some(c) if !is_ident_continue(c) => true,
28 _ => false,
29 };
30 if is_single_letter {
31 return if c == '_' { UNDERSCORE } else { IDENT };
32 }
25 ptr.bump_while(is_ident_continue); 33 ptr.bump_while(is_ident_continue);
26 return IDENT; 34 return IDENT;
27 } 35 }
diff --git a/tests/data/lexer/0003_ident.rs b/tests/data/lexer/0003_ident.rs
new file mode 100644
index 000000000..c05c9c009
--- /dev/null
+++ b/tests/data/lexer/0003_ident.rs
@@ -0,0 +1 @@
foo foo_ _foo _ __ x привет
diff --git a/tests/data/lexer/0003_ident.txt b/tests/data/lexer/0003_ident.txt
new file mode 100644
index 000000000..eec82fb91
--- /dev/null
+++ b/tests/data/lexer/0003_ident.txt
@@ -0,0 +1,14 @@
1IDENT 3
2WHITESPACE 1
3IDENT 4
4WHITESPACE 1
5IDENT 4
6WHITESPACE 1
7UNDERSCORE 1
8WHITESPACE 1
9IDENT 2
10WHITESPACE 1
11IDENT 1
12WHITESPACE 1
13IDENT 12
14WHITESPACE 1
diff --git a/validation.md b/validation.md
index 9cfec5309..3706760ba 100644
--- a/validation.md
+++ b/validation.md
@@ -1,4 +1,5 @@
1Fixmes: 1Fixmes:
2 2
3* Fix `is_whitespace`, add more test 3* Fix `is_whitespace`, add more test
4* Add more thorough tests for idents for XID_Start & XID_Continue
4 5