aboutsummaryrefslogtreecommitdiff
path: root/src/lexer/mod.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/lexer/mod.rs')
-rw-r--r--src/lexer/mod.rs23
1 files changed, 18 insertions, 5 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 3e49b1c2b..f46746bee 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -13,6 +13,9 @@ use self::numbers::scan_number;
13mod strings; 13mod strings;
14use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string}; 14use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string};
15 15
16mod comments;
17use self::comments::{scan_shebang, scan_comment};
18
16pub fn next_token(text: &str) -> Token { 19pub fn next_token(text: &str) -> Token {
17 assert!(!text.is_empty()); 20 assert!(!text.is_empty());
18 let mut ptr = Ptr::new(text); 21 let mut ptr = Ptr::new(text);
@@ -23,16 +26,26 @@ pub fn next_token(text: &str) -> Token {
23} 26}
24 27
25fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { 28fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
26 let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
27 if ident_start {
28 return scan_ident(c, ptr);
29 }
30
31 if is_whitespace(c) { 29 if is_whitespace(c) {
32 ptr.bump_while(is_whitespace); 30 ptr.bump_while(is_whitespace);
33 return WHITESPACE; 31 return WHITESPACE;
34 } 32 }
35 33
34 match c {
35 '#' => if scan_shebang(ptr) {
36 return SHEBANG;
37 }
38 '/' => if let Some(kind) = scan_comment(ptr) {
39 return kind;
40 }
41 _ => (),
42 }
43
44 let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
45 if ident_start {
46 return scan_ident(c, ptr);
47 }
48
36 if is_dec_digit(c) { 49 if is_dec_digit(c) {
37 let kind = scan_number(c, ptr); 50 let kind = scan_number(c, ptr);
38 scan_literal_suffix(ptr); 51 scan_literal_suffix(ptr);