From 9ce4db545efba697f20ab5cecbefc0589c7146ca Mon Sep 17 00:00:00 2001
From: Aleksey Kladov <aleksey.kladov@gmail.com>
Date: Sun, 31 Dec 2017 17:54:33 +0300
Subject: Parser: groundwork

---
 src/lexer/mod.rs | 11 +++++++++++
 1 file changed, 11 insertions(+)

(limited to 'src/lexer/mod.rs')

diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
index f46746bee..7c4259763 100644
--- a/src/lexer/mod.rs
+++ b/src/lexer/mod.rs
@@ -16,6 +16,17 @@ use self::strings::{is_string_literal_start, scan_char, scan_byte_char_or_string
 mod comments;
 use self::comments::{scan_shebang, scan_comment};
 
+pub fn tokenize(text: &str) -> Vec<Token> {
+    let mut text = text;
+    let mut acc = Vec::new();
+    while !text.is_empty() {
+        let token = next_token(text);
+        acc.push(token);
+        let len: u32 = token.len.into();
+        text = &text[len as usize..];
+    }
+    acc
+}
 pub fn next_token(text: &str) -> Token {
     assert!(!text.is_empty());
     let mut ptr = Ptr::new(text);
-- 
cgit v1.2.3