aboutsummaryrefslogtreecommitdiff
path: root/src/lexer
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2017-12-31 13:42:22 +0000
committerAleksey Kladov <[email protected]>2017-12-31 13:42:22 +0000
commitcb6f07618440859a26fd6adea63bd030da375952 (patch)
treef929dc7e20b7359f123b19f70243312a11473429 /src/lexer
parent9ccf339c176e68d26f601f24ed40bfa9429f9b5d (diff)
Lexer: comments groundwork
Diffstat (limited to 'src/lexer')
-rw-r--r--src/lexer/comments.rs11
-rw-r--r--src/lexer/mod.rs23
2 files changed, 29 insertions, 5 deletions
diff --git a/src/lexer/comments.rs b/src/lexer/comments.rs
new file mode 100644
index 000000000..c61c85824
--- /dev/null
+++ b/src/lexer/comments.rs
@@ -0,0 +1,11 @@
1use lexer::ptr::Ptr;
2
3use {SyntaxKind};
4
5pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool {
6 false
7}
8
9pub(crate) fn scan_comment(ptr: &mut Ptr) -> Option<SyntaxKind> {
10 None
11} \ No newline at end of file
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 3e49b1c2b..f46746bee 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -13,6 +13,9 @@ use self::numbers::scan_number;
13mod strings; 13mod strings;
14use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string}; 14use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string};
15 15
16mod comments;
17use self::comments::{scan_shebang, scan_comment};
18
16pub fn next_token(text: &str) -> Token { 19pub fn next_token(text: &str) -> Token {
17 assert!(!text.is_empty()); 20 assert!(!text.is_empty());
18 let mut ptr = Ptr::new(text); 21 let mut ptr = Ptr::new(text);
@@ -23,16 +26,26 @@ pub fn next_token(text: &str) -> Token {
23} 26}
24 27
25fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { 28fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
26 let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
27 if ident_start {
28 return scan_ident(c, ptr);
29 }
30
31 if is_whitespace(c) { 29 if is_whitespace(c) {
32 ptr.bump_while(is_whitespace); 30 ptr.bump_while(is_whitespace);
33 return WHITESPACE; 31 return WHITESPACE;
34 } 32 }
35 33
34 match c {
35 '#' => if scan_shebang(ptr) {
36 return SHEBANG;
37 }
38 '/' => if let Some(kind) = scan_comment(ptr) {
39 return kind;
40 }
41 _ => (),
42 }
43
44 let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
45 if ident_start {
46 return scan_ident(c, ptr);
47 }
48
36 if is_dec_digit(c) { 49 if is_dec_digit(c) {
37 let kind = scan_number(c, ptr); 50 let kind = scan_number(c, ptr);
38 scan_literal_suffix(ptr); 51 scan_literal_suffix(ptr);