aboutsummaryrefslogtreecommitdiff
path: root/src/lexer
diff options
context:
space:
mode:
authorAleksey Kladov <[email protected]>2017-12-31 14:54:33 +0000
committerAleksey Kladov <[email protected]>2017-12-31 14:54:33 +0000
commit9ce4db545efba697f20ab5cecbefc0589c7146ca (patch)
treeecda02126426b473bf2f70777a148e569c114c98 /src/lexer
parent98a58bf806ffda1b4d3352ed0f3e494fa25c8c74 (diff)
Parser: groundwork
Diffstat (limited to 'src/lexer')
-rw-r--r--src/lexer/mod.rs11
1 files changed, 11 insertions, 0 deletions
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index f46746bee..7c4259763 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -16,6 +16,17 @@ use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string
16mod comments; 16mod comments;
17use self::comments::{scan_shebang, scan_comment}; 17use self::comments::{scan_shebang, scan_comment};
18 18
19pub fn tokenize(text: &str) -> Vec<Token> {
20 let mut text = text;
21 let mut acc = Vec::new();
22 while !text.is_empty() {
23 let token = next_token(text);
24 acc.push(token);
25 let len: u32 = token.len.into();
26 text = &text[len as usize..];
27 }
28 acc
29}
19pub fn next_token(text: &str) -> Token { 30pub fn next_token(text: &str) -> Token {
20 assert!(!text.is_empty()); 31 assert!(!text.is_empty());
21 let mut ptr = Ptr::new(text); 32 let mut ptr = Ptr::new(text);