From ac37a11f04b31f792068a1cb50dbbf5ccd4d982d Mon Sep 17 00:00:00 2001 From: Veetaha Date: Sun, 26 Jan 2020 20:44:49 +0200 Subject: Reimplemented lexer with vectors instead of iterators --- crates/ra_syntax/src/parsing.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'crates/ra_syntax/src/parsing.rs') diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index 0387f0378..4e51f920b 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -7,15 +7,17 @@ mod text_tree_sink; mod reparsing; use crate::{syntax_node::GreenNode, SyntaxError}; +use text_token_source::TextTokenSource; +use text_tree_sink::TextTreeSink; -pub use self::lexer::{classify_literal, tokenize, Token}; +pub use lexer::*; pub(crate) use self::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { - let tokens = tokenize(&text); - let mut token_source = text_token_source::TextTokenSource::new(text, &tokens); - let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens); + let ParsedTokens { tokens, errors } = tokenize(&text); + let mut token_source = TextTokenSource::new(text, &tokens); + let mut tree_sink = TextTreeSink::new(text, &tokens, errors); ra_parser::parse(&mut token_source, &mut tree_sink); tree_sink.finish() } -- cgit v1.2.3 From 9e7eaa959f9dc368a55f1a80b35651b78b3d0883 Mon Sep 17 00:00:00 2001 From: Veetaha Date: Tue, 28 Jan 2020 07:09:13 +0200 Subject: ra_syntax: refactored the lexer design as per @matklad and @kiljacken PR review --- crates/ra_syntax/src/parsing.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'crates/ra_syntax/src/parsing.rs') diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index 4e51f920b..e5eb80850 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -15,9 +15,15 @@ pub use lexer::*; pub(crate) use self::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { - let ParsedTokens { tokens, errors } = tokenize(&text); + let (tokens, lexer_errors) = tokenize(&text); + let mut token_source = TextTokenSource::new(text, &tokens); - let mut tree_sink = TextTreeSink::new(text, &tokens, errors); + let mut tree_sink = TextTreeSink::new(text, &tokens); + ra_parser::parse(&mut token_source, &mut tree_sink); - tree_sink.finish() + + let (tree, mut parser_errors) = tree_sink.finish(); + parser_errors.extend(lexer_errors); + + (tree, parser_errors) } -- cgit v1.2.3