diff options
author | Aleksey Kladov <[email protected]> | 2018-07-29 13:16:07 +0100 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2018-07-29 13:16:07 +0100 |
commit | 415c891d641fa305e7ddbbbcc78db990dd5d3564 (patch) | |
tree | 4b6e1e0aa4b5a732aeae8945e75c9bee3bbf1d65 /src/lexer | |
parent | ad188d4c3db34f035408afbdd6d2f3c308121f0a (diff) |
Reorganize
Diffstat (limited to 'src/lexer')
-rw-r--r-- | src/lexer/comments.rs | 3 | ||||
-rw-r--r-- | src/lexer/mod.rs | 50 | ||||
-rw-r--r-- | src/lexer/numbers.rs | 3 | ||||
-rw-r--r-- | src/lexer/strings.rs | 3 |
4 files changed, 34 insertions, 25 deletions
diff --git a/src/lexer/comments.rs b/src/lexer/comments.rs index d1e958817..01acb6515 100644 --- a/src/lexer/comments.rs +++ b/src/lexer/comments.rs | |||
@@ -1,7 +1,6 @@ | |||
1 | use lexer::ptr::Ptr; | 1 | use lexer::ptr::Ptr; |
2 | 2 | ||
3 | use SyntaxKind; | 3 | use SyntaxKind::{self, *}; |
4 | use syntax_kinds::*; | ||
5 | 4 | ||
6 | pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool { | 5 | pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool { |
7 | if ptr.next_is('!') && ptr.nnext_is('/') { | 6 | if ptr.next_is('!') && ptr.nnext_is('/') { |
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 65a994327..69cab5b57 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs | |||
@@ -1,21 +1,32 @@ | |||
1 | use {SyntaxKind, Token}; | ||
2 | use syntax_kinds::*; | ||
3 | |||
4 | mod ptr; | 1 | mod ptr; |
5 | use self::ptr::Ptr; | 2 | mod comments; |
6 | 3 | mod strings; | |
4 | mod numbers; | ||
7 | mod classes; | 5 | mod classes; |
8 | use self::classes::*; | ||
9 | 6 | ||
10 | mod numbers; | 7 | use { |
11 | use self::numbers::scan_number; | 8 | TextUnit, |
9 | SyntaxKind::{self, *}, | ||
10 | }; | ||
12 | 11 | ||
13 | mod strings; | 12 | use self::{ |
14 | use self::strings::{is_string_literal_start, scan_byte_char_or_string, scan_char, scan_raw_string, | 13 | ptr::Ptr, |
15 | scan_string}; | 14 | classes::*, |
15 | numbers::scan_number, | ||
16 | strings::{ | ||
17 | is_string_literal_start, scan_byte_char_or_string, scan_char, | ||
18 | scan_raw_string, scan_string}, | ||
19 | comments::{scan_comment, scan_shebang}, | ||
20 | }; | ||
16 | 21 | ||
17 | mod comments; | 22 | /// A token of Rust source. |
18 | use self::comments::{scan_comment, scan_shebang}; | 23 | #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] |
24 | pub struct Token { | ||
25 | /// The kind of token. | ||
26 | pub kind: SyntaxKind, | ||
27 | /// The length of the token. | ||
28 | pub len: TextUnit, | ||
29 | } | ||
19 | 30 | ||
20 | /// Break a string up into its component tokens | 31 | /// Break a string up into its component tokens |
21 | pub fn tokenize(text: &str) -> Vec<Token> { | 32 | pub fn tokenize(text: &str) -> Vec<Token> { |
@@ -29,6 +40,7 @@ pub fn tokenize(text: &str) -> Vec<Token> { | |||
29 | } | 40 | } |
30 | acc | 41 | acc |
31 | } | 42 | } |
43 | |||
32 | /// Get the next token from a string | 44 | /// Get the next token from a string |
33 | pub fn next_token(text: &str) -> Token { | 45 | pub fn next_token(text: &str) -> Token { |
34 | assert!(!text.is_empty()); | 46 | assert!(!text.is_empty()); |
@@ -109,7 +121,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
109 | DOTDOT | 121 | DOTDOT |
110 | } | 122 | } |
111 | _ => DOT, | 123 | _ => DOT, |
112 | } | 124 | }; |
113 | } | 125 | } |
114 | ':' => { | 126 | ':' => { |
115 | return match ptr.next() { | 127 | return match ptr.next() { |
@@ -118,7 +130,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
118 | COLONCOLON | 130 | COLONCOLON |
119 | } | 131 | } |
120 | _ => COLON, | 132 | _ => COLON, |
121 | } | 133 | }; |
122 | } | 134 | } |
123 | '=' => { | 135 | '=' => { |
124 | return match ptr.next() { | 136 | return match ptr.next() { |
@@ -131,7 +143,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
131 | FAT_ARROW | 143 | FAT_ARROW |
132 | } | 144 | } |
133 | _ => EQ, | 145 | _ => EQ, |
134 | } | 146 | }; |
135 | } | 147 | } |
136 | '!' => { | 148 | '!' => { |
137 | return match ptr.next() { | 149 | return match ptr.next() { |
@@ -140,7 +152,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
140 | NEQ | 152 | NEQ |
141 | } | 153 | } |
142 | _ => EXCL, | 154 | _ => EXCL, |
143 | } | 155 | }; |
144 | } | 156 | } |
145 | '-' => { | 157 | '-' => { |
146 | return if ptr.next_is('>') { | 158 | return if ptr.next_is('>') { |
@@ -148,7 +160,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
148 | THIN_ARROW | 160 | THIN_ARROW |
149 | } else { | 161 | } else { |
150 | MINUS | 162 | MINUS |
151 | } | 163 | }; |
152 | } | 164 | } |
153 | 165 | ||
154 | // If the character is an ident start not followed by another single | 166 | // If the character is an ident start not followed by another single |
@@ -202,7 +214,7 @@ fn scan_ident(c: char, ptr: &mut Ptr) -> SyntaxKind { | |||
202 | return if c == '_' { UNDERSCORE } else { IDENT }; | 214 | return if c == '_' { UNDERSCORE } else { IDENT }; |
203 | } | 215 | } |
204 | ptr.bump_while(is_ident_continue); | 216 | ptr.bump_while(is_ident_continue); |
205 | if let Some(kind) = ident_to_keyword(ptr.current_token_text()) { | 217 | if let Some(kind) = SyntaxKind::from_keyword(ptr.current_token_text()) { |
206 | return kind; | 218 | return kind; |
207 | } | 219 | } |
208 | IDENT | 220 | IDENT |
diff --git a/src/lexer/numbers.rs b/src/lexer/numbers.rs index 95e42246f..38eac9212 100644 --- a/src/lexer/numbers.rs +++ b/src/lexer/numbers.rs | |||
@@ -1,8 +1,7 @@ | |||
1 | use lexer::ptr::Ptr; | 1 | use lexer::ptr::Ptr; |
2 | use lexer::classes::*; | 2 | use lexer::classes::*; |
3 | 3 | ||
4 | use SyntaxKind; | 4 | use SyntaxKind::{self, *}; |
5 | use syntax_kinds::*; | ||
6 | 5 | ||
7 | pub(crate) fn scan_number(c: char, ptr: &mut Ptr) -> SyntaxKind { | 6 | pub(crate) fn scan_number(c: char, ptr: &mut Ptr) -> SyntaxKind { |
8 | if c == '0' { | 7 | if c == '0' { |
diff --git a/src/lexer/strings.rs b/src/lexer/strings.rs index 00a84ec85..e3704fbb3 100644 --- a/src/lexer/strings.rs +++ b/src/lexer/strings.rs | |||
@@ -1,5 +1,4 @@ | |||
1 | use SyntaxKind; | 1 | use SyntaxKind::{self, *}; |
2 | use syntax_kinds::*; | ||
3 | 2 | ||
4 | use lexer::ptr::Ptr; | 3 | use lexer::ptr::Ptr; |
5 | 4 | ||