Reimplemented lexer with vectors instead of iterators

1 files changed, 6 insertions, 4 deletions

diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs
index 0387f0378..4e51f920b 100644
--- a/crates/ra_syntax/src/parsing.rs
+++ b/crates/ra_syntax/src/parsing.rs
@@ -7,15 +7,17 @@ mod text_tree_sink;
 mod reparsing;
 
 use crate::{syntax_node::GreenNode, SyntaxError};
+use text_token_source::TextTokenSource;
+use text_tree_sink::TextTreeSink;
 
-pub use self::lexer::{classify_literal, tokenize, Token};
+pub use lexer::*;
 
 pub(crate) use self::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
-    let tokens = tokenize(&text);
-    let mut token_source = text_token_source::TextTokenSource::new(text, &tokens);
-    let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens);
+    let ParsedTokens { tokens, errors } = tokenize(&text);
+    let mut token_source = TextTokenSource::new(text, &tokens);
+    let mut tree_sink = TextTreeSink::new(text, &tokens, errors);
     ra_parser::parse(&mut token_source, &mut tree_sink);
     tree_sink.finish()
 }