From fcb1eef3232c3fc673bf5f98595708e108c3950c Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 25 May 2019 20:31:53 +0800 Subject: Change TokenSource to iteration based --- crates/ra_mbe/src/subtree_parser.rs | 4 +-- crates/ra_mbe/src/subtree_source.rs | 59 ++++++++++++++++++++++++------------- crates/ra_mbe/src/syntax_bridge.rs | 42 +++++++++++++++----------- 3 files changed, 64 insertions(+), 41 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs index 709b87a38..9cc989b23 100644 --- a/crates/ra_mbe/src/subtree_parser.rs +++ b/crates/ra_mbe/src/subtree_parser.rs @@ -68,13 +68,13 @@ impl<'a> Parser<'a> { fn parse(self, f: F) -> Option where - F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), + F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink), { let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]); let mut src = SubtreeTokenSource::new(&buffer); let mut sink = OffsetTokenSink { token_pos: 0, error: false }; - f(&src, &mut sink); + f(&mut src, &mut sink); let r = self.finish(sink.token_pos, &mut src); if sink.error { diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 972af4a7c..c4f79f38a 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -1,6 +1,7 @@ -use ra_parser::{TokenSource}; +use ra_parser::{TokenSource, Token}; use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T}; use std::cell::{RefCell, Cell}; +use std::sync::Arc; use tt::buffer::{TokenBuffer, Cursor}; pub(crate) trait Querier { @@ -65,7 +66,7 @@ impl<'a> SubtreeWalk<'a> { return cached[pos].clone(); } - fn collect_token_trees(&mut self, n: usize) -> Vec { + fn collect_token_trees(&self, n: usize) -> Vec { let mut res = vec![]; let mut pos = 0; @@ -117,43 +118,59 @@ impl<'a> Querier for SubtreeWalk<'a> { } pub(crate) struct SubtreeTokenSource<'a> { - walker: SubtreeWalk<'a>, + walker: Arc>, + curr: (Token, usize), } impl<'a> SubtreeTokenSource<'a> { pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { - SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) } + let mut res = SubtreeTokenSource { + walker: Arc::new(SubtreeWalk::new(buffer.begin())), + curr: (Token { kind: EOF, is_jointed_to_next: false }, 0), + }; + res.curr = (res.mk_token(0), 0); + res } - pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a> - where - 'a: 'b, - { - &self.walker + pub fn querier(&self) -> Arc> { + self.walker.clone() } pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec { let res = self.walker.collect_token_trees(parsed_tokens); res } + + fn mk_token(&self, pos: usize) -> Token { + match self.walker.get(pos) { + Some(tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next }, + None => Token { kind: EOF, is_jointed_to_next: false }, + } + } } impl<'a> TokenSource for SubtreeTokenSource<'a> { - fn token_kind(&self, pos: usize) -> SyntaxKind { - if let Some(tok) = self.walker.get(pos) { - tok.kind - } else { - SyntaxKind::EOF - } + fn current(&self) -> Token { + self.curr.0 } - fn is_token_joint_to_next(&self, pos: usize) -> bool { - match self.walker.get(pos) { - Some(t) => t.is_joint_to_next, - _ => false, + + /// Lookahead n token + fn lookahead_nth(&self, n: usize) -> Token { + self.mk_token(self.curr.1 + n) + } + + /// bump cursor to next token + fn bump(&mut self) { + if self.current().kind == EOF { + return; } + + self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1) } - fn is_keyword(&self, pos: usize, kw: &str) -> bool { - match self.walker.get(pos) { + + /// Is the current token a specified keyword? + fn is_keyword(&self, kw: &str) -> bool { + match self.walker.get(self.curr.1) { Some(t) => t.text == *kw, _ => false, } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 0a75305b4..0aab5ea8b 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -48,9 +48,10 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke /// Parses the token tree (result of macro expansion) to an expression pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_expr(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_expr(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -64,9 +65,10 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, Expand /// Parses the token tree (result of macro expansion) to a Pattern pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_pat(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_pat(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -78,9 +80,10 @@ pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandEr /// Parses the token tree (result of macro expansion) to a Type pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_ty(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_ty(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -93,9 +96,10 @@ pub fn token_tree_to_macro_stmts( tt: &tt::Subtree, ) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_macro_stmts(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -108,9 +112,10 @@ pub fn token_tree_to_macro_items( tt: &tt::Subtree, ) -> Result, ExpandError> { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse_macro_items(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse_macro_items(&mut token_source, &mut tree_sink); if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } @@ -121,9 +126,10 @@ pub fn token_tree_to_macro_items( /// Parses the token tree (result of macro expansion) as a sequence of items pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc { let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let token_source = SubtreeTokenSource::new(&buffer); - let mut tree_sink = TtTreeSink::new(token_source.querier()); - ra_parser::parse(&token_source, &mut tree_sink); + let mut token_source = SubtreeTokenSource::new(&buffer); + let querier = token_source.querier(); + let mut tree_sink = TtTreeSink::new(querier.as_ref()); + ra_parser::parse(&mut token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); ast::SourceFile::cast(&syntax).unwrap().to_owned() } -- cgit v1.2.3 From 816147c4b508117fad4ca4d3cb6d77a3c4a683a9 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sat, 25 May 2019 21:55:46 +0800 Subject: Simplify token_tree_to_xxx --- crates/ra_mbe/src/syntax_bridge.rs | 67 ++++++++++++-------------------------- 1 file changed, 20 insertions(+), 47 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 0aab5ea8b..6265fe462 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -45,18 +45,29 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke // // -/// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { +fn token_tree_to_syntax_node(tt: &tt::Subtree, f: F) -> Result, ExpandError> +where + F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink), +{ let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); let mut token_source = SubtreeTokenSource::new(&buffer); let querier = token_source.querier(); let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_expr(&mut token_source, &mut tree_sink); + f(&mut token_source, &mut tree_sink); + if tree_sink.roots.len() != 1 { + return Err(ExpandError::ConversionError); + } + if tree_sink.roots.len() != 1 { return Err(ExpandError::ConversionError); } - let syntax = tree_sink.inner.finish(); + Ok(tree_sink.inner.finish()) +} + +/// Parses the token tree (result of macro expansion) to an expression +pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_expr)?; ast::Expr::cast(&syntax) .map(|m| m.to_owned()) .ok_or_else(|| crate::ExpandError::ConversionError) @@ -64,30 +75,13 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, Expand /// Parses the token tree (result of macro expansion) to a Pattern pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_pat(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_pat)?; ast::Pat::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) to a Type pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_ty(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_ty)?; ast::TypeRef::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } @@ -95,15 +89,7 @@ pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, Expan pub fn token_tree_to_macro_stmts( tt: &tt::Subtree, ) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_macro_stmts(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_stmts)?; ast::MacroStmts::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } @@ -111,26 +97,13 @@ pub fn token_tree_to_macro_stmts( pub fn token_tree_to_macro_items( tt: &tt::Subtree, ) -> Result, ExpandError> { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse_macro_items(&mut token_source, &mut tree_sink); - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_items)?; ast::MacroItems::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) as a sequence of items pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc { - let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]); - let mut token_source = SubtreeTokenSource::new(&buffer); - let querier = token_source.querier(); - let mut tree_sink = TtTreeSink::new(querier.as_ref()); - ra_parser::parse(&mut token_source, &mut tree_sink); - let syntax = tree_sink.inner.finish(); + let syntax = token_tree_to_syntax_node(tt, ra_parser::parse).unwrap(); ast::SourceFile::cast(&syntax).unwrap().to_owned() } -- cgit v1.2.3 From 90764fc54b2be1e0fc5d6ac9c9e960d7bb059b14 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Sun, 26 May 2019 02:41:00 +0800 Subject: Remove duplicated code --- crates/ra_mbe/src/syntax_bridge.rs | 4 ---- 1 file changed, 4 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 6265fe462..0a7e50c4e 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -58,10 +58,6 @@ where return Err(ExpandError::ConversionError); } - if tree_sink.roots.len() != 1 { - return Err(ExpandError::ConversionError); - } - Ok(tree_sink.inner.finish()) } -- cgit v1.2.3