aboutsummaryrefslogtreecommitdiff
path: root/src/lexer
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2018-07-29 13:16:07 +0100
committerAleksey Kladov <[email protected]>2018-07-29 13:16:07 +0100
commit415c891d641fa305e7ddbbbcc78db990dd5d3564 (patch)
tree4b6e1e0aa4b5a732aeae8945e75c9bee3bbf1d65 /src/lexer
parentad188d4c3db34f035408afbdd6d2f3c308121f0a (diff)
Reorganize
Diffstat (limited to 'src/lexer')
-rw-r--r--src/lexer/comments.rs3
-rw-r--r--src/lexer/mod.rs50
-rw-r--r--src/lexer/numbers.rs3
-rw-r--r--src/lexer/strings.rs3
4 files changed, 34 insertions, 25 deletions
diff --git a/src/lexer/comments.rs b/src/lexer/comments.rs
index d1e958817..01acb6515 100644
--- a/src/lexer/comments.rs
+++ b/src/lexer/comments.rs
@@ -1,7 +1,6 @@
1use lexer::ptr::Ptr; 1use lexer::ptr::Ptr;
2 2
3use SyntaxKind; 3use SyntaxKind::{self, *};
4use syntax_kinds::*;
5 4
6pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool { 5pub(crate) fn scan_shebang(ptr: &mut Ptr) -> bool {
7 if ptr.next_is('!') && ptr.nnext_is('/') { 6 if ptr.next_is('!') && ptr.nnext_is('/') {
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 65a994327..69cab5b57 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -1,21 +1,32 @@
1use {SyntaxKind, Token};
2use syntax_kinds::*;
3
4mod ptr; 1mod ptr;
5use self::ptr::Ptr; 2mod comments;
6 3mod strings;
4mod numbers;
7mod classes; 5mod classes;
8use self::classes::*;
9 6
10mod numbers; 7use {
11use self::numbers::scan_number; 8 TextUnit,
9 SyntaxKind::{self, *},
10};
12 11
13mod strings; 12use self::{
14use self::strings::{is_string_literal_start, scan_byte_char_or_string, scan_char, scan_raw_string, 13 ptr::Ptr,
15 scan_string}; 14 classes::*,
15 numbers::scan_number,
16 strings::{
17 is_string_literal_start, scan_byte_char_or_string, scan_char,
18 scan_raw_string, scan_string},
19 comments::{scan_comment, scan_shebang},
20};
16 21
17mod comments; 22/// A token of Rust source.
18use self::comments::{scan_comment, scan_shebang}; 23#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
24pub struct Token {
25 /// The kind of token.
26 pub kind: SyntaxKind,
27 /// The length of the token.
28 pub len: TextUnit,
29}
19 30
20/// Break a string up into its component tokens 31/// Break a string up into its component tokens
21pub fn tokenize(text: &str) -> Vec<Token> { 32pub fn tokenize(text: &str) -> Vec<Token> {
@@ -29,6 +40,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
29 } 40 }
30 acc 41 acc
31} 42}
43
32/// Get the next token from a string 44/// Get the next token from a string
33pub fn next_token(text: &str) -> Token { 45pub fn next_token(text: &str) -> Token {
34 assert!(!text.is_empty()); 46 assert!(!text.is_empty());
@@ -109,7 +121,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
109 DOTDOT 121 DOTDOT
110 } 122 }
111 _ => DOT, 123 _ => DOT,
112 } 124 };
113 } 125 }
114 ':' => { 126 ':' => {
115 return match ptr.next() { 127 return match ptr.next() {
@@ -118,7 +130,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
118 COLONCOLON 130 COLONCOLON
119 } 131 }
120 _ => COLON, 132 _ => COLON,
121 } 133 };
122 } 134 }
123 '=' => { 135 '=' => {
124 return match ptr.next() { 136 return match ptr.next() {
@@ -131,7 +143,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
131 FAT_ARROW 143 FAT_ARROW
132 } 144 }
133 _ => EQ, 145 _ => EQ,
134 } 146 };
135 } 147 }
136 '!' => { 148 '!' => {
137 return match ptr.next() { 149 return match ptr.next() {
@@ -140,7 +152,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
140 NEQ 152 NEQ
141 } 153 }
142 _ => EXCL, 154 _ => EXCL,
143 } 155 };
144 } 156 }
145 '-' => { 157 '-' => {
146 return if ptr.next_is('>') { 158 return if ptr.next_is('>') {
@@ -148,7 +160,7 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind {
148 THIN_ARROW 160 THIN_ARROW
149 } else { 161 } else {
150 MINUS 162 MINUS
151 } 163 };
152 } 164 }
153 165
154 // If the character is an ident start not followed by another single 166 // If the character is an ident start not followed by another single
@@ -202,7 +214,7 @@ fn scan_ident(c: char, ptr: &mut Ptr) -> SyntaxKind {
202 return if c == '_' { UNDERSCORE } else { IDENT }; 214 return if c == '_' { UNDERSCORE } else { IDENT };
203 } 215 }
204 ptr.bump_while(is_ident_continue); 216 ptr.bump_while(is_ident_continue);
205 if let Some(kind) = ident_to_keyword(ptr.current_token_text()) { 217 if let Some(kind) = SyntaxKind::from_keyword(ptr.current_token_text()) {
206 return kind; 218 return kind;
207 } 219 }
208 IDENT 220 IDENT
diff --git a/src/lexer/numbers.rs b/src/lexer/numbers.rs
index 95e42246f..38eac9212 100644
--- a/src/lexer/numbers.rs
+++ b/src/lexer/numbers.rs
@@ -1,8 +1,7 @@
1use lexer::ptr::Ptr; 1use lexer::ptr::Ptr;
2use lexer::classes::*; 2use lexer::classes::*;
3 3
4use SyntaxKind; 4use SyntaxKind::{self, *};
5use syntax_kinds::*;
6 5
7pub(crate) fn scan_number(c: char, ptr: &mut Ptr) -> SyntaxKind { 6pub(crate) fn scan_number(c: char, ptr: &mut Ptr) -> SyntaxKind {
8 if c == '0' { 7 if c == '0' {
diff --git a/src/lexer/strings.rs b/src/lexer/strings.rs
index 00a84ec85..e3704fbb3 100644
--- a/src/lexer/strings.rs
+++ b/src/lexer/strings.rs
@@ -1,5 +1,4 @@
1use SyntaxKind; 1use SyntaxKind::{self, *};
2use syntax_kinds::*;
3 2
4use lexer::ptr::Ptr; 3use lexer::ptr::Ptr;
5 4