Parser: groundwork

1 files changed, 1 insertions, 13 deletions

diff --git a/tests/lexer.rs b/tests/lexer.rs
index 6a9bab66b..beca19c24 100644
--- a/tests/lexer.rs
+++ b/tests/lexer.rs
@@ -7,7 +7,7 @@ use std::path::{PathBuf, Path};
 use std::fs::read_dir;
 use std::fmt::Write;
 
-use libsyntax2::{Token, next_token};
+use libsyntax2::{Token, tokenize};
 
 #[test]
 fn lexer_tests() {
@@ -53,18 +53,6 @@ fn lexer_test_case(path: &Path) {
     assert_diff!(expected, actual, "\n", 0)
 }
 
-fn tokenize(text: &str) -> Vec<Token> {
-    let mut text = text;
-    let mut acc = Vec::new();
-    while !text.is_empty() {
-        let token = next_token(text);
-        acc.push(token);
-        let len: u32 = token.len.into();
-        text = &text[len as usize..];
-    }
-    acc
-}
-
 fn dump_tokens(tokens: &[Token], text: &str) -> String {
     let mut acc = String::new();
     let mut offset = 0;