aboutsummaryrefslogtreecommitdiff
path: root/src/lexer
diff options
context:
space:
mode:
Diffstat (limited to 'src/lexer')
-rw-r--r--src/lexer/mod.rs2
1 files changed, 2 insertions, 0 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index 2f8d3a402..65a994327 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -17,6 +17,7 @@ use self::strings::{is_string_literal_start, scan_byte_char_or_string, scan_char
17mod comments; 17mod comments;
18use self::comments::{scan_comment, scan_shebang}; 18use self::comments::{scan_comment, scan_shebang};
19 19
20/// Break a string up into its component tokens
20pub fn tokenize(text: &str) -> Vec<Token> { 21pub fn tokenize(text: &str) -> Vec<Token> {
21 let mut text = text; 22 let mut text = text;
22 let mut acc = Vec::new(); 23 let mut acc = Vec::new();
@@ -28,6 +29,7 @@ pub fn tokenize(text: &str) -> Vec<Token> {
28 } 29 }
29 acc 30 acc
30} 31}
32/// Get the next token from a string
31pub fn next_token(text: &str) -> Token { 33pub fn next_token(text: &str) -> Token {
32 assert!(!text.is_empty()); 34 assert!(!text.is_empty());
33 let mut ptr = Ptr::new(text); 35 let mut ptr = Ptr::new(text);