aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2017-12-31 13:42:22 +0000
committerAleksey Kladov <[email protected]>2017-12-31 13:42:22 +0000
commitcb6f07618440859a26fd6adea63bd030da375952 (patch)
treef929dc7e20b7359f123b19f70243312a11473429 /src
parent9ccf339c176e68d26f601f24ed40bfa9429f9b5d (diff)
Lexer: comments groundwork
Diffstat (limited to 'src')
-rw-r--r--src/lexer/comments.rs11
-rw-r--r--src/lexer/mod.rs23
-rw-r--r--src/syntax_kinds.rs8
3 files changed, 36 insertions, 6 deletions
diff --git a/src/lexer/comments.rs b/src/lexer/comments.rs
new file mode 100644
index 000000000..c61c85824
--- /dev/null
+++ b/src/lexer/comments.rs
@@ -0,0 +1,11 @@
1use lexer::ptr::Ptr;
2
3use {SyntaxKind};
4
5pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool {
6 false
7}
8
9pub(crate) fn scan_comment(ptr: &mut Ptr) -> Option<SyntaxKind> {
10 None
11} \ No newline at end of file
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 3e49b1c2b..f46746bee 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -13,6 +13,9 @@ use self::numbers::scan_number;
13mod strings; 13mod strings;
14use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string}; 14use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string};
15 15
16mod comments;
17use self::comments::{scan_shebang, scan_comment};
18
16pub fn next_token(text: &str) -> Token { 19pub fn next_token(text: &str) -> Token {
17 assert!(!text.is_empty()); 20 assert!(!text.is_empty());
18 let mut ptr = Ptr::new(text); 21 let mut ptr = Ptr::new(text);
@@ -23,16 +26,26 @@ pub fn next_token(text: &str) -> Token {
23} 26}
24 27
25fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { 28fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
26 let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
27 if ident_start {
28 return scan_ident(c, ptr);
29 }
30
31 if is_whitespace(c) { 29 if is_whitespace(c) {
32 ptr.bump_while(is_whitespace); 30 ptr.bump_while(is_whitespace);
33 return WHITESPACE; 31 return WHITESPACE;
34 } 32 }
35 33
34 match c {
35 '#' => if scan_shebang(ptr) {
36 return SHEBANG;
37 }
38 '/' => if let Some(kind) = scan_comment(ptr) {
39 return kind;
40 }
41 _ => (),
42 }
43
44 let ident_start = is_ident_start(c) && !is_string_literal_start(c, ptr.next(), ptr.nnext());
45 if ident_start {
46 return scan_ident(c, ptr);
47 }
48
36 if is_dec_digit(c) { 49 if is_dec_digit(c) {
37 let kind = scan_number(c, ptr); 50 let kind = scan_number(c, ptr);
38 scan_literal_suffix(ptr); 51 scan_literal_suffix(ptr);
diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs
index 83fabe403..ec2a036b9 100644
--- a/src/syntax_kinds.rs
+++ b/src/syntax_kinds.rs
@@ -49,8 +49,11 @@ pub const PERCENT: SyntaxKind = SyntaxKind(44);
49pub const AMPERSAND: SyntaxKind = SyntaxKind(45); 49pub const AMPERSAND: SyntaxKind = SyntaxKind(45);
50pub const PIPE: SyntaxKind = SyntaxKind(46); 50pub const PIPE: SyntaxKind = SyntaxKind(46);
51pub const THIN_ARROW: SyntaxKind = SyntaxKind(47); 51pub const THIN_ARROW: SyntaxKind = SyntaxKind(47);
52pub const COMMENT: SyntaxKind = SyntaxKind(48);
53pub const DOC_COMMENT: SyntaxKind = SyntaxKind(49);
54pub const SHEBANG: SyntaxKind = SyntaxKind(50);
52 55
53static INFOS: [SyntaxInfo; 48] = [ 56static INFOS: [SyntaxInfo; 51] = [
54 SyntaxInfo { name: "ERROR" }, 57 SyntaxInfo { name: "ERROR" },
55 SyntaxInfo { name: "IDENT" }, 58 SyntaxInfo { name: "IDENT" },
56 SyntaxInfo { name: "UNDERSCORE" }, 59 SyntaxInfo { name: "UNDERSCORE" },
@@ -99,6 +102,9 @@ static INFOS: [SyntaxInfo; 48] = [
99 SyntaxInfo { name: "AMPERSAND" }, 102 SyntaxInfo { name: "AMPERSAND" },
100 SyntaxInfo { name: "PIPE" }, 103 SyntaxInfo { name: "PIPE" },
101 SyntaxInfo { name: "THIN_ARROW" }, 104 SyntaxInfo { name: "THIN_ARROW" },
105 SyntaxInfo { name: "COMMENT" },
106 SyntaxInfo { name: "DOC_COMMENT" },
107 SyntaxInfo { name: "SHEBANG" },
102]; 108];
103 109
104pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { 110pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo {