From 9ce4db545efba697f20ab5cecbefc0589c7146ca Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 31 Dec 2017 17:54:33 +0300 Subject: Parser: groundwork --- tests/lexer.rs | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'tests/lexer.rs') diff --git a/tests/lexer.rs b/tests/lexer.rs index 6a9bab66b..beca19c24 100644 --- a/tests/lexer.rs +++ b/tests/lexer.rs @@ -7,7 +7,7 @@ use std::path::{PathBuf, Path}; use std::fs::read_dir; use std::fmt::Write; -use libsyntax2::{Token, next_token}; +use libsyntax2::{Token, tokenize}; #[test] fn lexer_tests() { @@ -53,18 +53,6 @@ fn lexer_test_case(path: &Path) { assert_diff!(expected, actual, "\n", 0) } -fn tokenize(text: &str) -> Vec { - let mut text = text; - let mut acc = Vec::new(); - while !text.is_empty() { - let token = next_token(text); - acc.push(token); - let len: u32 = token.len.into(); - text = &text[len as usize..]; - } - acc -} - fn dump_tokens(tokens: &[Token], text: &str) -> String { let mut acc = String::new(); let mut offset = 0; -- cgit v1.2.3