From 4551182f94fe81c314f79ddf8916a5520cfd03b0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 17 Sep 2019 02:54:22 +0300 Subject: use usual token tree for macro expansion --- crates/ra_mbe/src/lib.rs | 157 +++++------ crates/ra_mbe/src/mbe_expander.rs | 19 +- crates/ra_mbe/src/mbe_expander/matcher.rs | 361 +++++++++++++++++--------- crates/ra_mbe/src/mbe_expander/transcriber.rs | 288 ++++++++++---------- crates/ra_mbe/src/mbe_parser.rs | 187 ------------- crates/ra_mbe/src/parser.rs | 187 +++++++++++++ crates/ra_mbe/src/subtree_parser.rs | 91 ------- crates/ra_mbe/src/tests.rs | 48 ++++ crates/ra_mbe/src/tt_cursor.rs | 281 -------------------- crates/ra_mbe/src/tt_iter.rs | 67 +++++ 10 files changed, 753 insertions(+), 933 deletions(-) delete mode 100644 crates/ra_mbe/src/mbe_parser.rs create mode 100644 crates/ra_mbe/src/parser.rs delete mode 100644 crates/ra_mbe/src/subtree_parser.rs delete mode 100644 crates/ra_mbe/src/tt_cursor.rs create mode 100644 crates/ra_mbe/src/tt_iter.rs diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index f07f000ff..41720df79 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -3,30 +3,19 @@ /// interface, although it contains some code to bridge `SyntaxNode`s and /// `TokenTree`s as well! -macro_rules! impl_froms { - ($e:ident: $($v:ident), *) => { - $( - impl From<$v> for $e { - fn from(it: $v) -> $e { - $e::$v(it) - } - } - )* - } -} - -mod mbe_parser; +mod parser; mod mbe_expander; mod syntax_bridge; -mod tt_cursor; +mod tt_iter; mod subtree_source; -mod subtree_parser; - -use ra_syntax::SmolStr; -use smallvec::SmallVec; pub use tt::{Delimiter, Punct}; +use crate::{ + parser::{parse_pattern, Op}, + tt_iter::TtIter, +}; + #[derive(Debug, PartialEq, Eq)] pub enum ParseError { Expected(String), @@ -38,6 +27,7 @@ pub enum ExpandError { UnexpectedToken, BindingError(String), ConversionError, + InvalidRepeat, } pub use crate::syntax_bridge::{ @@ -54,97 +44,72 @@ pub struct MacroRules { pub(crate) rules: Vec, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Rule { + pub(crate) lhs: tt::Subtree, + pub(crate) rhs: tt::Subtree, +} + impl MacroRules { pub fn parse(tt: &tt::Subtree) -> Result { - mbe_parser::parse(tt) + let mut src = TtIter::new(tt); + let mut rules = Vec::new(); + while src.len() > 0 { + let rule = Rule::parse(&mut src)?; + rules.push(rule); + if let Err(()) = src.expect_char(';') { + if src.len() > 0 { + return Err(ParseError::Expected("expected `:`".to_string())); + } + break; + } + } + Ok(MacroRules { rules }) } pub fn expand(&self, tt: &tt::Subtree) -> Result { mbe_expander::expand(self, tt) } } -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Rule { - pub(crate) lhs: Subtree, - pub(crate) rhs: Subtree, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum TokenTree { - Leaf(Leaf), - Subtree(Subtree), - Repeat(Repeat), -} -impl_froms!(TokenTree: Leaf, Subtree, Repeat); - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum Leaf { - Literal(Literal), - Punct(Punct), - Ident(Ident), - Var(Var), -} -impl_froms!(Leaf: Literal, Punct, Ident, Var); - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Subtree { - pub(crate) delimiter: Delimiter, - pub(crate) token_trees: Vec, -} - -#[derive(Clone, Debug, Eq)] -pub(crate) enum Separator { - Literal(tt::Literal), - Ident(tt::Ident), - Puncts(SmallVec<[tt::Punct; 3]>), +impl Rule { + fn parse(src: &mut TtIter) -> Result { + let mut lhs = src + .expect_subtree() + .map_err(|()| ParseError::Expected("expected subtree".to_string()))? + .clone(); + validate(&lhs)?; + lhs.delimiter = tt::Delimiter::None; + src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?; + src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?; + let mut rhs = src + .expect_subtree() + .map_err(|()| ParseError::Expected("expected subtree".to_string()))? + .clone(); + rhs.delimiter = tt::Delimiter::None; + Ok(crate::Rule { lhs, rhs }) + } } -// Note that when we compare a Separator, we just care about its textual value. -impl PartialEq for crate::Separator { - fn eq(&self, other: &crate::Separator) -> bool { - use crate::Separator::*; - - match (self, other) { - (Ident(ref a), Ident(ref b)) => a.text == b.text, - (Literal(ref a), Literal(ref b)) => a.text == b.text, - (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { - let a_iter = a.iter().map(|a| a.char); - let b_iter = b.iter().map(|b| b.char); - a_iter.eq(b_iter) +fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> { + for op in parse_pattern(pattern) { + let op = match op { + Ok(it) => it, + Err(e) => { + let msg = match e { + ExpandError::InvalidRepeat => "invalid repeat".to_string(), + _ => "invalid macro definition".to_string(), + }; + return Err(ParseError::Expected(msg)); + } + }; + match op { + Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => { + validate(subtree)? } - _ => false, + _ => (), } } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Repeat { - pub(crate) subtree: Subtree, - pub(crate) kind: RepeatKind, - pub(crate) separator: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum RepeatKind { - ZeroOrMore, - OneOrMore, - ZeroOrOne, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Literal { - pub(crate) text: SmolStr, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Ident { - pub(crate) text: SmolStr, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct Var { - pub(crate) text: SmolStr, - pub(crate) kind: Option, + Ok(()) } #[cfg(test)] diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 15d9d83e2..b455b7321 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -8,7 +8,6 @@ mod transcriber; use ra_syntax::SmolStr; use rustc_hash::FxHashMap; -use crate::tt_cursor::TtCursor; use crate::ExpandError; pub(crate) fn expand( @@ -19,12 +18,8 @@ pub(crate) fn expand( } fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result { - let mut input = TtCursor::new(input); - let bindings = matcher::match_lhs(&rule.lhs, &mut input)?; - if !input.is_eof() { - return Err(ExpandError::UnexpectedToken); - } - let res = transcriber::transcribe(&bindings, &rule.rhs)?; + let bindings = matcher::match_(&rule.lhs, input)?; + let res = transcriber::transcribe(&rule.rhs, &bindings)?; Ok(res) } @@ -103,13 +98,6 @@ mod tests { #[test] fn test_expand_rule() { - // FIXME: The missing $var check should be in parsing phase - // assert_err( - // "($i:ident) => ($j)", - // "foo!{a}", - // ExpandError::BindingError(String::from("could not find binding `j`")), - // ); - assert_err( "($($i:ident);*) => ($i)", "foo!{a}", @@ -118,9 +106,6 @@ mod tests { )), ); - assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken); - assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken); - // FIXME: // Add an err test case for ($($i:ident)) => ($()) } diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 100a3b0e0..aff953102 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -1,11 +1,14 @@ use crate::{ mbe_expander::{Binding, Bindings, Fragment}, - tt_cursor::TtCursor, + parser::{parse_pattern, Op, RepeatKind, Separator}, + subtree_source::SubtreeTokenSource, + tt_iter::TtIter, ExpandError, }; -use ra_parser::FragmentKind::*; -use ra_syntax::SmolStr; +use ra_parser::{FragmentKind::*, TreeSink}; +use ra_syntax::{SmolStr, SyntaxKind}; +use tt::buffer::{Cursor, TokenBuffer}; impl Bindings { fn push_optional(&mut self, name: &SmolStr) { @@ -42,121 +45,247 @@ impl Bindings { } Ok(()) } +} - fn merge(&mut self, nested: Bindings) { - self.inner.extend(nested.inner); - } +macro_rules! err { + () => { + ExpandError::BindingError(format!("")) + }; + ($($tt:tt)*) => { + ExpandError::BindingError(format!($($tt)*)) + }; } -pub(super) fn match_lhs( - pattern: &crate::Subtree, - input: &mut TtCursor, -) -> Result { +macro_rules! bail { + ($($tt:tt)*) => { + return Err(err!($($tt)*)) + }; +} + +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result { + assert!(pattern.delimiter == tt::Delimiter::None); + let mut res = Bindings::default(); - for pat in pattern.token_trees.iter() { - match pat { - crate::TokenTree::Leaf(leaf) => match leaf { - crate::Leaf::Var(crate::Var { text, kind }) => { - let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; - match match_meta_var(kind.as_str(), input)? { - Some(fragment) => { - res.inner.insert(text.clone(), Binding::Fragment(fragment)); - } - None => res.push_optional(text), - } + let mut src = TtIter::new(src); + + match_subtree(&mut res, pattern, &mut src)?; + + if src.len() > 0 { + bail!("leftover tokens"); + } + + Ok(res) +} + +fn match_subtree( + bindings: &mut Bindings, + pattern: &tt::Subtree, + src: &mut TtIter, +) -> Result<(), ExpandError> { + for op in parse_pattern(pattern) { + match op? { + Op::TokenTree(tt::TokenTree::Leaf(lhs)) => { + let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?; + match (lhs, rhs) { + ( + tt::Leaf::Punct(tt::Punct { char: lhs, .. }), + tt::Leaf::Punct(tt::Punct { char: rhs, .. }), + ) if lhs == rhs => (), + ( + tt::Leaf::Ident(tt::Ident { text: lhs, .. }), + tt::Leaf::Ident(tt::Ident { text: rhs, .. }), + ) if lhs == rhs => (), + ( + tt::Leaf::Literal(tt::Literal { text: lhs, .. }), + tt::Leaf::Literal(tt::Literal { text: rhs, .. }), + ) if lhs == rhs => (), + _ => Err(ExpandError::UnexpectedToken)?, } - crate::Leaf::Punct(punct) => { - if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) { - return Err(ExpandError::UnexpectedToken); - } + } + Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { + let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; + if lhs.delimiter != rhs.delimiter { + bail!("mismatched delimiter") } - crate::Leaf::Ident(ident) => { - if input.eat_ident().map(|i| &i.text) != Some(&ident.text) { - return Err(ExpandError::UnexpectedToken); - } + let mut src = TtIter::new(rhs); + match_subtree(bindings, lhs, &mut src)?; + if src.len() > 0 { + bail!("leftover tokens"); } - crate::Leaf::Literal(literal) => { - if input.eat_literal().map(|i| &i.text) != Some(&literal.text) { - return Err(ExpandError::UnexpectedToken); + } + Op::Var { name, kind } => { + let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; + match match_meta_var(kind.as_str(), src)? { + Some(fragment) => { + bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); } + None => bindings.push_optional(name), } + () + } + Op::Repeat { subtree, kind, separator } => { + match_repeat(bindings, subtree, kind, separator, src)? + } + } + } + Ok(()) +} + +impl<'a> TtIter<'a> { + fn eat_separator(&mut self, separator: &Separator) -> bool { + let mut fork = self.clone(); + let ok = match separator { + Separator::Ident(lhs) => match fork.expect_ident() { + Ok(rhs) => rhs.text == lhs.text, + _ => false, }, - crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => { - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a propper macro-by-example implementation - let mut limit = 65536; - let mut counter = 0; - - let mut memento = input.save(); - - loop { - match match_lhs(subtree, input) { - Ok(nested) => { - limit -= 1; - if limit == 0 { - log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); - break; - } - - memento = input.save(); - res.push_nested(counter, nested)?; - counter += 1; - if counter == 1 { - if let crate::RepeatKind::ZeroOrOne = kind { - break; - } - } - - if let Some(separator) = separator { - if !input - .eat_seperator() - .map(|sep| sep == *separator) - .unwrap_or(false) - { - input.rollback(memento); - break; - } - } - } - Err(_) => { - input.rollback(memento); - break; - } - } - } + Separator::Literal(lhs) => match fork.expect_literal() { + Ok(rhs) => rhs.text == lhs.text, + _ => false, + }, + Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { + Ok(rhs) => rhs.char == lhs.char, + _ => false, + }), + }; + if ok { + *self = fork; + } + ok + } - match kind { - crate::RepeatKind::OneOrMore if counter == 0 => { - return Err(ExpandError::UnexpectedToken); - } - _ if counter == 0 => { - // Collect all empty variables in subtrees - collect_vars(subtree).iter().for_each(|s| res.push_empty(s)); - } - _ => {} + pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> { + let ident = self.expect_ident()?; + // check if it start from "`" + if ident.text.chars().next() != Some('\'') { + return Err(()); + } + Ok(ident) + } + + pub(crate) fn expect_fragment( + &mut self, + fragment_kind: ra_parser::FragmentKind, + ) -> Result { + pub(crate) struct OffsetTokenSink<'a> { + pub(crate) cursor: Cursor<'a>, + pub(crate) error: bool, + } + + impl<'a> TreeSink for OffsetTokenSink<'a> { + fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { + for _ in 0..n_tokens { + self.cursor = self.cursor.bump_subtree(); } } - crate::TokenTree::Subtree(subtree) => { - let input_subtree = - input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?; - if subtree.delimiter != input_subtree.delimiter { - return Err(ExpandError::UnexpectedToken); + fn start_node(&mut self, _kind: SyntaxKind) {} + fn finish_node(&mut self) {} + fn error(&mut self, _error: ra_parser::ParseError) { + self.error = true; + } + } + + let buffer = TokenBuffer::new(self.inner.as_slice()); + let mut src = SubtreeTokenSource::new(&buffer); + let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; + + ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind); + + if !sink.cursor.is_root() || sink.error { + return Err(()); + } + + let mut curr = buffer.begin(); + let mut res = vec![]; + + while curr != sink.cursor { + if let Some(token) = curr.token_tree() { + res.push(token); + } + curr = curr.bump(); + } + self.inner = self.inner.as_slice()[res.len()..].iter(); + match res.len() { + 0 => Err(()), + 1 => Ok(res[0].clone()), + _ => Ok(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::None, + token_trees: res.into_iter().cloned().collect(), + })), + } + } + + pub(crate) fn eat_vis(&mut self) -> Option { + let mut fork = self.clone(); + match fork.expect_fragment(Visibility) { + Ok(tt) => { + *self = fork; + Some(tt) + } + Err(()) => None, + } + } +} + +pub(super) fn match_repeat( + bindings: &mut Bindings, + pattern: &tt::Subtree, + kind: RepeatKind, + separator: Option, + src: &mut TtIter, +) -> Result<(), ExpandError> { + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a propper macro-by-example implementation + let mut limit = 65536; + let mut counter = 0; + + for i in 0.. { + let mut fork = src.clone(); + + if let Some(separator) = &separator { + if i != 0 && !fork.eat_separator(separator) { + break; + } + } + + let mut nested = Bindings::default(); + match match_subtree(&mut nested, pattern, &mut fork) { + Ok(()) => { + limit -= 1; + if limit == 0 { + log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator); + break; } + *src = fork; - let mut input = TtCursor::new(input_subtree); - let bindings = match_lhs(&subtree, &mut input)?; - if !input.is_eof() { - return Err(ExpandError::UnexpectedToken); + bindings.push_nested(counter, nested)?; + counter += 1; + if counter == 1 { + if let RepeatKind::ZeroOrOne = kind { + break; + } } + } + Err(_) => break, + } + } - res.merge(bindings); + match (kind, counter) { + (RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken), + (_, 0) => { + // Collect all empty variables in subtrees + let mut vars = Vec::new(); + collect_vars(&mut vars, pattern)?; + for var in vars { + bindings.push_empty(&var) } } + _ => (), } - Ok(res) + Ok(()) } -fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result, ExpandError> { +fn match_meta_var(kind: &str, input: &mut TtIter) -> Result, ExpandError> { let fragment = match kind { "path" => Path, "expr" => Expr, @@ -169,17 +298,20 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result, _ => { let tt = match kind { "ident" => { - let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone(); + let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone(); tt::Leaf::from(ident).into() } - "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(), - "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(), + "tt" => input.next().ok_or_else(|| err!())?.clone(), + "lifetime" => { + let ident = input.expect_lifetime().map_err(|()| err!())?; + tt::Leaf::Ident(ident.clone()).into() + } "literal" => { - let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); + let literal = input.expect_literal().map_err(|()| err!())?.clone(); tt::Leaf::from(literal).into() } // `vis` is optional - "vis" => match input.try_eat_vis() { + "vis" => match input.eat_vis() { Some(vis) => vis, None => return Ok(None), }, @@ -188,28 +320,19 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result, return Ok(Some(Fragment::Tokens(tt))); } }; - let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?; + let tt = input.expect_fragment(fragment).map_err(|()| err!())?; let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; Ok(Some(fragment)) } -fn collect_vars(subtree: &crate::Subtree) -> Vec { - let mut res = Vec::new(); - - for tkn in subtree.token_trees.iter() { - match tkn { - crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => { - res.push(text.clone()); - } - crate::TokenTree::Subtree(subtree) => { - res.extend(collect_vars(subtree)); - } - crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => { - res.extend(collect_vars(subtree)); - } - _ => {} +fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), ExpandError> { + for op in parse_pattern(pattern) { + match op? { + Op::Var { name, .. } => buf.push(name.clone()), + Op::TokenTree(tt::TokenTree::Leaf(_)) => (), + Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?, + Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?, } } - - res + Ok(()) } diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index a3df1b7de..c22680b93 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -1,16 +1,20 @@ +//! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like +//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` + use ra_syntax::SmolStr; use crate::{ mbe_expander::{Binding, Bindings, Fragment}, + parser::{parse_template, Op, RepeatKind, Separator}, ExpandError, }; impl Bindings { - fn contains(&self, name: &SmolStr) -> bool { + fn contains(&self, name: &str) -> bool { self.inner.contains_key(name) } - fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { + fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> { let mut b = self.inner.get(name).ok_or_else(|| { ExpandError::BindingError(format!("could not find binding `{}`", name)) })?; @@ -43,11 +47,12 @@ impl Bindings { } pub(super) fn transcribe( + template: &tt::Subtree, bindings: &Bindings, - template: &crate::Subtree, ) -> Result { + assert!(template.delimiter == tt::Delimiter::None); let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; - expand_subtree(template, &mut ctx) + expand_subtree(&mut ctx, template) } #[derive(Debug)] @@ -57,159 +62,158 @@ struct ExpandCtx<'a> { var_expanded: bool, } -fn expand_subtree( - template: &crate::Subtree, - ctx: &mut ExpandCtx, -) -> Result { +fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result { let mut buf: Vec = Vec::new(); - for tt in template.token_trees.iter() { - let tt = expand_tt(tt, ctx)?; - push_fragment(&mut buf, tt); + for op in parse_template(template) { + match op? { + Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), + Op::TokenTree(tt::TokenTree::Subtree(tt)) => { + let tt = expand_subtree(ctx, tt)?; + buf.push(tt.into()); + } + Op::Var { name, kind: _ } => { + let fragment = expand_var(ctx, name)?; + push_fragment(&mut buf, fragment); + } + Op::Repeat { subtree, kind, separator } => { + let fragment = expand_repeat(ctx, subtree, kind, separator)?; + push_fragment(&mut buf, fragment) + } + } } - Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) } -fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result { - let res: tt::TokenTree = match template { - crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), - crate::TokenTree::Repeat(repeat) => { - let mut buf: Vec = Vec::new(); - ctx.nesting.push(0); - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a propper macro-by-example implementation - let mut limit = 65536; - let mut has_seps = 0; - let mut counter = 0; - - // We store the old var expanded value, and restore it later - // It is because before this `$repeat`, - // it is possible some variables already expanad in the same subtree - // - // `some_var_expanded` keep check if the deeper subtree has expanded variables - let mut some_var_expanded = false; - let old_var_expanded = ctx.var_expanded; - ctx.var_expanded = false; - - while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { - // if no var expanded in the child, we count it as a fail - if !ctx.var_expanded { - break; - } +fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result { + let res = if v == "crate" { + // FIXME: Properly handle $crate token + let tt = + tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) + .into(); + Fragment::Tokens(tt) + } else if !ctx.bindings.contains(v) { + // Note that it is possible to have a `$var` inside a macro which is not bound. + // For example: + // ``` + // macro_rules! foo { + // ($a:ident, $b:ident, $c:tt) => { + // macro_rules! bar { + // ($bi:ident) => { + // fn $bi() -> u8 {$c} + // } + // } + // } + // ``` + // We just treat it a normal tokens + let tt = tt::Subtree { + delimiter: tt::Delimiter::None, + token_trees: vec![ + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), + tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) + .into(), + ], + } + .into(); + Fragment::Tokens(tt) + } else { + let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone(); + ctx.var_expanded = true; + fragment + }; + Ok(res) +} - // Reset `ctx.var_expandeded` to see if there is other expanded variable - // in the next matching - some_var_expanded = true; - ctx.var_expanded = false; - - counter += 1; - limit -= 1; - if limit == 0 { - log::warn!( - "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", - template, - ctx - ); - break; - } +fn expand_repeat( + ctx: &mut ExpandCtx, + template: &tt::Subtree, + kind: RepeatKind, + separator: Option, +) -> Result { + let mut buf: Vec = Vec::new(); + ctx.nesting.push(0); + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a propper macro-by-example implementation + let mut limit = 65536; + let mut has_seps = 0; + let mut counter = 0; + + // We store the old var expanded value, and restore it later + // It is because before this `$repeat`, + // it is possible some variables already expanad in the same subtree + // + // `some_var_expanded` keep check if the deeper subtree has expanded variables + let mut some_var_expanded = false; + let old_var_expanded = ctx.var_expanded; + ctx.var_expanded = false; + + while let Ok(mut t) = expand_subtree(ctx, template) { + t.delimiter = tt::Delimiter::None; + // if no var expanded in the child, we count it as a fail + if !ctx.var_expanded { + break; + } - let idx = ctx.nesting.pop().unwrap(); - ctx.nesting.push(idx + 1); - push_subtree(&mut buf, t); - - if let Some(ref sep) = repeat.separator { - match sep { - crate::Separator::Ident(ident) => { - has_seps = 1; - buf.push(tt::Leaf::from(ident.clone()).into()); - } - crate::Separator::Literal(lit) => { - has_seps = 1; - buf.push(tt::Leaf::from(lit.clone()).into()); - } - - crate::Separator::Puncts(puncts) => { - has_seps = puncts.len(); - for punct in puncts { - buf.push(tt::Leaf::from(*punct).into()); - } - } - } + // Reset `ctx.var_expandeded` to see if there is other expanded variable + // in the next matching + some_var_expanded = true; + ctx.var_expanded = false; + + counter += 1; + limit -= 1; + if limit == 0 { + log::warn!( + "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", + template, + ctx + ); + break; + } + + let idx = ctx.nesting.pop().unwrap(); + ctx.nesting.push(idx + 1); + push_subtree(&mut buf, t); + + if let Some(ref sep) = separator { + match sep { + Separator::Ident(ident) => { + has_seps = 1; + buf.push(tt::Leaf::from(ident.clone()).into()); + } + Separator::Literal(lit) => { + has_seps = 1; + buf.push(tt::Leaf::from(lit.clone()).into()); } - if let crate::RepeatKind::ZeroOrOne = repeat.kind { - break; + Separator::Puncts(puncts) => { + has_seps = puncts.len(); + for punct in puncts { + buf.push(tt::Leaf::from(*punct).into()); + } } } + } - // Restore the `var_expanded` by combining old one and the new one - ctx.var_expanded = some_var_expanded || old_var_expanded; + if RepeatKind::ZeroOrOne == kind { + break; + } + } - ctx.nesting.pop().unwrap(); - for _ in 0..has_seps { - buf.pop(); - } + // Restore the `var_expanded` by combining old one and the new one + ctx.var_expanded = some_var_expanded || old_var_expanded; - if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { - return Err(ExpandError::UnexpectedToken); - } + ctx.nesting.pop().unwrap(); + for _ in 0..has_seps { + buf.pop(); + } - // Check if it is a single token subtree without any delimiter - // e.g {Delimiter:None> ['>'] /Delimiter:None>} - tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() - } - crate::TokenTree::Leaf(leaf) => match leaf { - crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident { - text: ident.text.clone(), - id: tt::TokenId::unspecified(), - }) - .into(), - crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(), - crate::Leaf::Var(v) => { - if v.text == "crate" { - // FIXME: Properly handle $crate token - tt::Leaf::from(tt::Ident { - text: "$crate".into(), - id: tt::TokenId::unspecified(), - }) - .into() - } else if !ctx.bindings.contains(&v.text) { - // Note that it is possible to have a `$var` inside a macro which is not bound. - // For example: - // ``` - // macro_rules! foo { - // ($a:ident, $b:ident, $c:tt) => { - // macro_rules! bar { - // ($bi:ident) => { - // fn $bi() -> u8 {$c} - // } - // } - // } - // ``` - // We just treat it a normal tokens - tt::Subtree { - delimiter: tt::Delimiter::None, - token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }) - .into(), - tt::Leaf::from(tt::Ident { - text: v.text.clone(), - id: tt::TokenId::unspecified(), - }) - .into(), - ], - } - .into() - } else { - let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); - ctx.var_expanded = true; - return Ok(fragment); - } - } - crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(), - }, - }; - Ok(Fragment::Tokens(res)) + if RepeatKind::OneOrMore == kind && counter == 0 { + return Err(ExpandError::UnexpectedToken); + } + + // Check if it is a single token subtree without any delimiter + // e.g {Delimiter:None> ['>'] /Delimiter:None>} + let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into(); + Ok(Fragment::Tokens(tt)) } fn push_fragment(buf: &mut Vec, fragment: Fragment) { diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs deleted file mode 100644 index 954b84d9d..000000000 --- a/crates/ra_mbe/src/mbe_parser.rs +++ /dev/null @@ -1,187 +0,0 @@ -use crate::tt_cursor::TtCursor; -/// This module parses a raw `tt::TokenStream` into macro-by-example token -/// stream. This is a *mostly* identify function, expect for handling of -/// `$var:tt_kind` and `$(repeat),*` constructs. -use crate::ParseError; - -pub(crate) fn parse(tt: &tt::Subtree) -> Result { - let mut parser = TtCursor::new(tt); - let mut rules = Vec::new(); - while !parser.is_eof() { - rules.push(parse_rule(&mut parser)?); - if let Err(e) = parser.expect_char(';') { - if !parser.is_eof() { - return Err(e); - } - break; - } - } - Ok(crate::MacroRules { rules }) -} - -fn parse_rule(p: &mut TtCursor) -> Result { - let lhs = parse_subtree(p.eat_subtree()?, false)?; - p.expect_char('=')?; - p.expect_char('>')?; - let mut rhs = parse_subtree(p.eat_subtree()?, true)?; - rhs.delimiter = crate::Delimiter::None; - Ok(crate::Rule { lhs, rhs }) -} - -fn is_boolean_literal(lit: Option<&tt::TokenTree>) -> bool { - if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = lit { - if lit.text == "true" || lit.text == "false" { - return true; - } - } - - false -} - -fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result { - let mut token_trees = Vec::new(); - let mut p = TtCursor::new(tt); - while let Some(tt) = p.eat() { - let child: crate::TokenTree = match tt { - tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Punct(tt::Punct { char: '$', spacing }) => { - // mbe var can be an ident or keyword, including `true` and `false` - if p.at_ident().is_some() || is_boolean_literal(p.current()) { - crate::Leaf::from(parse_var(&mut p, transcriber)?).into() - } else if let Some(tt::TokenTree::Subtree(_)) = p.current() { - parse_repeat(&mut p, transcriber)?.into() - } else { - // Treat it as normal punct - crate::Leaf::from(tt::Punct { char: '$', spacing: *spacing }).into() - } - } - tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(), - tt::Leaf::Ident(tt::Ident { text, .. }) => { - crate::Leaf::from(crate::Ident { text: text.clone() }).into() - } - tt::Leaf::Literal(tt::Literal { text }) => { - crate::Leaf::from(crate::Literal { text: text.clone() }).into() - } - }, - tt::TokenTree::Subtree(subtree) => parse_subtree(&subtree, transcriber)?.into(), - }; - token_trees.push(child); - } - Ok(crate::Subtree { token_trees, delimiter: tt.delimiter }) -} - -fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result { - let text = { - if is_boolean_literal(p.current()) { - let lit = p.eat_literal().unwrap(); - lit.text.clone() - } else { - let ident = p.eat_ident().unwrap(); - ident.text.clone() - } - }; - - let kind = if !transcriber && p.at_char(':') { - p.bump(); - if let Some(ident) = p.eat_ident() { - Some(ident.text.clone()) - } else { - p.rev_bump(); - None - } - } else { - None - }; - - Ok(crate::Var { text, kind }) -} - -fn mk_repeat( - rep: char, - subtree: crate::Subtree, - separator: Option, -) -> Result { - let kind = match rep { - '*' => crate::RepeatKind::ZeroOrMore, - '+' => crate::RepeatKind::OneOrMore, - '?' => crate::RepeatKind::ZeroOrOne, - _ => return Err(ParseError::Expected(String::from("repeat"))), - }; - Ok(crate::Repeat { subtree, kind, separator }) -} - -fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result { - let subtree = p.eat_subtree()?; - let mut subtree = parse_subtree(subtree, transcriber)?; - subtree.delimiter = crate::Delimiter::None; - - if let Some(rep) = p.at_punct() { - match rep.char { - '*' | '+' | '?' => { - p.bump(); - return mk_repeat(rep.char, subtree, None); - } - _ => {} - } - } - - let sep = p.eat_seperator().ok_or_else(|| ParseError::Expected(String::from("separator")))?; - let rep = p.eat_punct().ok_or_else(|| ParseError::Expected(String::from("repeat")))?; - - mk_repeat(rep.char, subtree, Some(sep)) -} - -#[cfg(test)] -mod tests { - use ra_syntax::{ast, AstNode}; - - use super::*; - use crate::ast_to_token_tree; - - #[test] - fn test_invalid_parse() { - expect_err("invalid", "subtree"); - - is_valid("($i:ident) => ()"); - is_valid("($($i:ident)*) => ($_)"); - is_valid("($($true:ident)*) => ($true)"); - is_valid("($($false:ident)*) => ($false)"); - - expect_err("$i:ident => ()", "subtree"); - expect_err("($i:ident) ()", "`=`"); - expect_err("($($i:ident)_) => ()", "repeat"); - } - - fn expect_err(macro_body: &str, expected: &str) { - assert_eq!( - create_rules(&format_macro(macro_body)), - Err(ParseError::Expected(String::from(expected))) - ); - } - - fn is_valid(macro_body: &str) { - assert!(create_rules(&format_macro(macro_body)).is_ok()); - } - - fn format_macro(macro_body: &str) -> String { - format!( - " - macro_rules! foo {{ - {} - }} -", - macro_body - ) - } - - fn create_rules(macro_definition: &str) -> Result { - let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap(); - let macro_definition = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (definition_tt, _) = - ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); - parse(&definition_tt) - } - -} diff --git a/crates/ra_mbe/src/parser.rs b/crates/ra_mbe/src/parser.rs new file mode 100644 index 000000000..575f587cf --- /dev/null +++ b/crates/ra_mbe/src/parser.rs @@ -0,0 +1,187 @@ +//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token +//! trees. + +use ra_syntax::SmolStr; +use smallvec::SmallVec; + +use crate::{tt_iter::TtIter, ExpandError}; + +#[derive(Debug)] +pub(crate) enum Op<'a> { + Var { name: &'a SmolStr, kind: Option<&'a SmolStr> }, + Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option }, + TokenTree(&'a tt::TokenTree), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum RepeatKind { + ZeroOrMore, + OneOrMore, + ZeroOrOne, +} + +#[derive(Clone, Debug, Eq)] +pub(crate) enum Separator { + Literal(tt::Literal), + Ident(tt::Ident), + Puncts(SmallVec<[tt::Punct; 3]>), +} + +// Note that when we compare a Separator, we just care about its textual value. +impl PartialEq for Separator { + fn eq(&self, other: &Separator) -> bool { + use Separator::*; + + match (self, other) { + (Ident(ref a), Ident(ref b)) => a.text == b.text, + (Literal(ref a), Literal(ref b)) => a.text == b.text, + (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { + let a_iter = a.iter().map(|a| a.char); + let b_iter = b.iter().map(|b| b.char); + a_iter.eq(b_iter) + } + _ => false, + } + } +} + +pub(crate) fn parse_template<'a>( + template: &'a tt::Subtree, +) -> impl Iterator, ExpandError>> { + parse_inner(template, Mode::Template) +} + +pub(crate) fn parse_pattern<'a>( + pattern: &'a tt::Subtree, +) -> impl Iterator, ExpandError>> { + parse_inner(pattern, Mode::Pattern) +} + +#[derive(Clone, Copy)] +enum Mode { + Pattern, + Template, +} + +fn parse_inner<'a>( + src: &'a tt::Subtree, + mode: Mode, +) -> impl Iterator, ExpandError>> { + let mut src = TtIter::new(src); + std::iter::from_fn(move || { + let first = src.next()?; + Some(next_op(first, &mut src, mode)) + }) +} + +macro_rules! err { + ($($tt:tt)*) => { + ExpandError::UnexpectedToken + }; +} + +macro_rules! bail { + ($($tt:tt)*) => { + return Err(err!($($tt)*)) + }; +} + +fn next_op<'a>( + first: &'a tt::TokenTree, + src: &mut TtIter<'a>, + mode: Mode, +) -> Result, ExpandError> { + let res = match first { + tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => { + let second = src.next().ok_or_else(|| err!("bad var 1"))?; + match second { + tt::TokenTree::Subtree(subtree) => { + let (separator, kind) = parse_repeat(src)?; + Op::Repeat { subtree, separator, kind } + } + tt::TokenTree::Leaf(leaf) => match leaf { + tt::Leaf::Punct(..) => Err(ExpandError::UnexpectedToken)?, + tt::Leaf::Ident(ident) => { + let name = &ident.text; + let kind = eat_fragment_kind(src, mode)?; + Op::Var { name, kind } + } + tt::Leaf::Literal(lit) => { + if is_boolean_literal(lit) { + let name = &lit.text; + let kind = eat_fragment_kind(src, mode)?; + Op::Var { name, kind } + } else { + bail!("bad var 2"); + } + } + }, + } + } + tt => Op::TokenTree(tt), + }; + Ok(res) +} + +fn eat_fragment_kind<'a>( + src: &mut TtIter<'a>, + mode: Mode, +) -> Result, ExpandError> { + if let Mode::Pattern = mode { + src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?; + let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?; + return Ok(Some(&ident.text)); + }; + Ok(None) +} + +fn is_boolean_literal(lit: &tt::Literal) -> bool { + match lit.text.as_str() { + "true" | "false" => true, + _ => false, + } +} + +///TOOD: impl for slice iter +fn parse_repeat(src: &mut TtIter) -> Result<(Option, RepeatKind), ExpandError> { + let mut separator = Separator::Puncts(SmallVec::new()); + for tt in src { + let tt = match tt { + tt::TokenTree::Leaf(leaf) => leaf, + tt::TokenTree::Subtree(_) => Err(ExpandError::InvalidRepeat)?, + }; + let has_sep = match &separator { + Separator::Puncts(puncts) => puncts.len() != 0, + _ => true, + }; + match tt { + tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => { + Err(ExpandError::InvalidRepeat)? + } + tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()), + tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()), + tt::Leaf::Punct(punct) => { + let repeat_kind = match punct.char { + '*' => RepeatKind::ZeroOrMore, + '+' => RepeatKind::OneOrMore, + '?' => RepeatKind::ZeroOrOne, + _ => { + match &mut separator { + Separator::Puncts(puncts) => { + if puncts.len() == 3 { + Err(ExpandError::InvalidRepeat)? + } + puncts.push(punct.clone()) + } + _ => Err(ExpandError::InvalidRepeat)?, + } + continue; + } + }; + let separator = if has_sep { Some(separator) } else { None }; + return Ok((separator, repeat_kind)); + } + } + } + Err(ExpandError::InvalidRepeat) +} diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs deleted file mode 100644 index 4440c69ff..000000000 --- a/crates/ra_mbe/src/subtree_parser.rs +++ /dev/null @@ -1,91 +0,0 @@ -use crate::subtree_source::SubtreeTokenSource; - -use ra_parser::{FragmentKind, TokenSource, TreeSink}; -use ra_syntax::SyntaxKind; -use tt::buffer::{Cursor, TokenBuffer}; - -struct OffsetTokenSink<'a> { - cursor: Cursor<'a>, - error: bool, -} - -impl<'a> OffsetTokenSink<'a> { - pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> { - if !self.cursor.is_root() { - return vec![]; - } - - let mut curr = begin; - let mut res = vec![]; - - while self.cursor != curr { - if let Some(token) = curr.token_tree() { - res.push(token); - } - curr = curr.bump(); - } - - res - } -} - -impl<'a> TreeSink for OffsetTokenSink<'a> { - fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { - for _ in 0..n_tokens { - self.cursor = self.cursor.bump_subtree(); - } - } - fn start_node(&mut self, _kind: SyntaxKind) {} - fn finish_node(&mut self) {} - fn error(&mut self, _error: ra_parser::ParseError) { - self.error = true; - } -} - -pub(crate) struct Parser<'a> { - subtree: &'a tt::Subtree, - cur_pos: &'a mut usize, -} - -impl<'a> Parser<'a> { - pub fn new(cur_pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> { - Parser { cur_pos, subtree } - } - - pub fn parse_fragment(self, fragment_kind: FragmentKind) -> Option { - self.parse(|token_source, tree_skink| { - ra_parser::parse_fragment(token_source, tree_skink, fragment_kind) - }) - } - - fn parse(self, f: F) -> Option - where - F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink), - { - let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]); - let mut src = SubtreeTokenSource::new(&buffer); - let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; - - f(&mut src, &mut sink); - - let r = self.finish(buffer.begin(), &mut sink); - if sink.error { - return None; - } - r - } - - fn finish(self, begin: Cursor, sink: &mut OffsetTokenSink) -> Option { - let res = sink.collect(begin); - *self.cur_pos += res.len(); - - match res.len() { - 0 => None, - 1 => Some(res[0].clone()), - _ => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::None, - token_trees: res.into_iter().cloned().collect(), - })), - } - } -} diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 312fa4626..d7482c63d 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -3,6 +3,54 @@ use test_utils::assert_eq_text; use super::*; +mod rule_parsing { + use ra_syntax::{ast, AstNode}; + + use super::*; + use crate::ast_to_token_tree; + + #[test] + fn test_valid_arms() { + fn check(macro_body: &str) { + let m = parse_macro_arm(macro_body); + m.unwrap(); + } + + check("($i:ident) => ()"); + check("($($i:ident)*) => ($_)"); + check("($($true:ident)*) => ($true)"); + check("($($false:ident)*) => ($false)"); + } + + #[test] + fn test_invalid_arms() { + fn check(macro_body: &str, err: &str) { + let m = parse_macro_arm(macro_body); + assert_eq!(m, Err(ParseError::Expected(String::from(err)))); + } + + check("invalid", "expected subtree"); + + check("$i:ident => ()", "expected subtree"); + check("($i:ident) ()", "expected `=`"); + check("($($i:ident)_) => ()", "invalid repeat"); + + check("($i) => ($i)", "invalid macro definition"); + check("($i:) => ($i)", "invalid macro definition"); + } + + fn parse_macro_arm(arm_definition: &str) -> Result { + let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition); + let source_file = ast::SourceFile::parse(¯o_definition).ok().unwrap(); + let macro_definition = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (definition_tt, _) = + ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); + crate::MacroRules::parse(&definition_tt) + } +} + // Good first issue (although a slightly challenging one): // // * Pick a random test from here diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs deleted file mode 100644 index a69c006c7..000000000 --- a/crates/ra_mbe/src/tt_cursor.rs +++ /dev/null @@ -1,281 +0,0 @@ -use crate::{subtree_parser::Parser, ParseError}; - -use ra_parser::FragmentKind; -use smallvec::{smallvec, SmallVec}; - -#[derive(Debug, Clone)] -pub(crate) struct TtCursor<'a> { - subtree: &'a tt::Subtree, - pos: usize, -} - -pub(crate) struct TtCursorMemento { - pos: usize, -} - -impl<'a> TtCursor<'a> { - pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> { - TtCursor { subtree, pos: 0 } - } - - pub(crate) fn is_eof(&self) -> bool { - self.pos == self.subtree.token_trees.len() - } - - pub(crate) fn current(&self) -> Option<&'a tt::TokenTree> { - self.subtree.token_trees.get(self.pos) - } - - pub(crate) fn at_punct(&self) -> Option<&'a tt::Punct> { - match self.current() { - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(it))) => Some(it), - _ => None, - } - } - - pub(crate) fn at_char(&self, char: char) -> bool { - match self.at_punct() { - Some(tt::Punct { char: c, .. }) if *c == char => true, - _ => false, - } - } - - pub(crate) fn at_ident(&mut self) -> Option<&'a tt::Ident> { - match self.current() { - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(i))) => Some(i), - _ => None, - } - } - - pub(crate) fn at_literal(&mut self) -> Option<&'a tt::Literal> { - match self.current() { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(i))) => Some(i), - _ => None, - } - } - - pub(crate) fn bump(&mut self) { - self.pos += 1; - } - pub(crate) fn rev_bump(&mut self) { - self.pos -= 1; - } - - pub(crate) fn eat(&mut self) -> Option<&'a tt::TokenTree> { - self.current().map(|it| { - self.bump(); - it - }) - } - - pub(crate) fn eat_subtree(&mut self) -> Result<&'a tt::Subtree, ParseError> { - match self.current() { - Some(tt::TokenTree::Subtree(sub)) => { - self.bump(); - Ok(sub) - } - _ => Err(ParseError::Expected(String::from("subtree"))), - } - } - - pub(crate) fn eat_punct(&mut self) -> Option<&'a tt::Punct> { - self.at_punct().map(|it| { - self.bump(); - it - }) - } - - pub(crate) fn eat_ident(&mut self) -> Option<&'a tt::Ident> { - self.at_ident().map(|i| { - self.bump(); - i - }) - } - - pub(crate) fn eat_literal(&mut self) -> Option<&'a tt::Literal> { - self.at_literal().map(|i| { - self.bump(); - i - }) - } - - pub(crate) fn eat_fragment(&mut self, fragment_kind: FragmentKind) -> Option { - let parser = Parser::new(&mut self.pos, self.subtree); - parser.parse_fragment(fragment_kind) - } - - pub(crate) fn eat_lifetime(&mut self) -> Option { - // check if it start from "`" - if let Some(ident) = self.at_ident() { - if ident.text.chars().next()? != '\'' { - return None; - } - } - - self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into()) - } - - pub(crate) fn try_eat_vis(&mut self) -> Option { - // `vis` matcher is optional - let old_pos = self.pos; - let parser = Parser::new(&mut self.pos, self.subtree); - - let res = parser.parse_fragment(FragmentKind::Visibility); - if res.is_none() { - self.pos = old_pos; - } - res - } - - pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { - if self.at_char(char) { - self.bump(); - Ok(()) - } else { - Err(ParseError::Expected(format!("`{}`", char))) - } - } - - fn eat_punct3(&mut self, p: tt::Punct) -> Option> { - let sec = *self.eat_punct()?; - let third = *self.eat_punct()?; - Some(smallvec![p, sec, third]) - } - - fn eat_punct2(&mut self, p: tt::Punct) -> Option> { - let sec = *self.eat_punct()?; - Some(smallvec![p, sec]) - } - - fn eat_multi_char_punct<'b, I>( - &mut self, - p: tt::Punct, - iter: &mut TokenPeek<'b, I>, - ) -> Option> - where - I: Iterator, - { - if let Some((m, _)) = iter.current_punct3(p) { - if let r @ Some(_) = match m { - ('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => { - self.eat_punct3(p) - } - _ => None, - } { - return r; - } - } - - if let Some((m, _)) = iter.current_punct2(p) { - if let r @ Some(_) = match m { - ('<', '=') - | ('>', '=') - | ('+', '=') - | ('-', '=') - | ('|', '=') - | ('&', '=') - | ('^', '=') - | ('/', '=') - | ('*', '=') - | ('%', '=') - | ('&', '&') - | ('|', '|') - | ('<', '<') - | ('>', '>') - | ('-', '>') - | ('!', '=') - | ('=', '>') - | ('=', '=') - | ('.', '.') - | (':', ':') => self.eat_punct2(p), - - _ => None, - } { - return r; - } - } - - None - } - - pub(crate) fn eat_seperator(&mut self) -> Option { - match self.eat()? { - tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - Some(crate::Separator::Literal(lit.clone())) - } - tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { - Some(crate::Separator::Ident(ident.clone())) - } - tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { - match punct.char { - '*' | '+' | '?' => return None, - _ => {} - }; - - // FIXME: The parser is only handle some compositeable punct, - // But at this phase, some punct still is jointed. - // So we by pass that check here. - let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter()); - let puncts = self.eat_multi_char_punct(*punct, &mut peekable); - let puncts = puncts.unwrap_or_else(|| smallvec![*punct]); - - Some(crate::Separator::Puncts(puncts)) - } - _ => None, - } - } - - #[must_use] - pub(crate) fn save(&self) -> TtCursorMemento { - TtCursorMemento { pos: self.pos } - } - - pub(crate) fn rollback(&mut self, memento: TtCursorMemento) { - self.pos = memento.pos; - } -} - -pub(crate) struct TokenPeek<'a, I> -where - I: Iterator, -{ - iter: itertools::MultiPeek, -} - -// helper function -fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> { - if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt { - return Some(pp); - } - None -} - -impl<'a, I> TokenPeek<'a, I> -where - I: Iterator, -{ - pub fn new(iter: I) -> Self { - TokenPeek { iter: itertools::multipeek(iter) } - } - - pub fn current_punct2(&mut self, p: tt::Punct) -> Option<((char, char), bool)> { - if p.spacing != tt::Spacing::Joint { - return None; - } - - self.iter.reset_peek(); - let p1 = to_punct(self.iter.peek()?)?; - Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint)) - } - - pub fn current_punct3(&mut self, p: tt::Punct) -> Option<((char, char, char), bool)> { - self.current_punct2(p).and_then(|((p0, p1), last_joint)| { - if !last_joint { - None - } else { - let p2 = to_punct(*self.iter.peek()?)?; - Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint)) - } - }) - } -} diff --git a/crates/ra_mbe/src/tt_iter.rs b/crates/ra_mbe/src/tt_iter.rs new file mode 100644 index 000000000..c53f99d1e --- /dev/null +++ b/crates/ra_mbe/src/tt_iter.rs @@ -0,0 +1,67 @@ +#[derive(Debug, Clone)] +pub(crate) struct TtIter<'a> { + pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, +} + +impl<'a> TtIter<'a> { + pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> { + TtIter { inner: subtree.token_trees.iter() } + } + + pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> { + match self.next() { + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => { + Ok(()) + } + _ => Err(()), + } + } + + pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { + match self.next() { + Some(tt::TokenTree::Subtree(it)) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { + match self.next() { + Some(tt::TokenTree::Leaf(it)) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { + match self.expect_leaf()? { + tt::Leaf::Ident(it) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Literal, ()> { + match self.expect_leaf()? { + tt::Leaf::Literal(it) => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> { + match self.expect_leaf()? { + tt::Leaf::Punct(it) => Ok(it), + _ => Err(()), + } + } +} + +impl<'a> Iterator for TtIter<'a> { + type Item = &'a tt::TokenTree; + fn next(&mut self) -> Option { + self.inner.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {} -- cgit v1.2.3