From b973158aeb337041d4e1434cf5d8c609a0b02bef Mon Sep 17 00:00:00 2001 From: Florian Diebold Date: Fri, 13 Mar 2020 13:03:31 +0100 Subject: Make MBE expansion more resilient (WIP) --- crates/ra_mbe/src/lib.rs | 4 +- crates/ra_mbe/src/mbe_expander.rs | 29 +++++++---- crates/ra_mbe/src/mbe_expander/matcher.rs | 11 +++-- crates/ra_mbe/src/mbe_expander/transcriber.rs | 69 ++++++++++++++++----------- crates/ra_mbe/src/tests.rs | 3 +- 5 files changed, 71 insertions(+), 45 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 43afe24cc..3adec4978 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -30,6 +30,8 @@ pub enum ExpandError { InvalidRepeat, } +pub type ExpandResult = (T, Option); + pub use crate::syntax_bridge::{ ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, TokenMap, @@ -150,7 +152,7 @@ impl MacroRules { Ok(MacroRules { rules, shift: Shift::new(tt) }) } - pub fn expand(&self, tt: &tt::Subtree) -> Result { + pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult { // apply shift let mut tt = tt.clone(); self.shift.shift_all(&mut tt); diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index b455b7321..da3952428 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -8,19 +8,30 @@ mod transcriber; use ra_syntax::SmolStr; use rustc_hash::FxHashMap; -use crate::ExpandError; +use crate::{ExpandResult, ExpandError}; pub(crate) fn expand( rules: &crate::MacroRules, input: &tt::Subtree, -) -> Result { - rules.rules.iter().find_map(|it| expand_rule(it, input).ok()).ok_or(ExpandError::NoMatchingRule) +) -> ExpandResult { + let (mut result, mut err) = (tt::Subtree::default(), Some(ExpandError::NoMatchingRule)); + for rule in &rules.rules { + let (res, e) = expand_rule(rule, input); + if e.is_none() { + // if we find a rule that applies without errors, we're done + return (res, None); + } + // TODO decide which result is better + result = res; + err = e; + } + (result, err) } -fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result { - let bindings = matcher::match_(&rule.lhs, input)?; - let res = transcriber::transcribe(&rule.rhs, &bindings)?; - Ok(res) +fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> ExpandResult { + let (bindings, bindings_err) = dbg!(matcher::match_(&rule.lhs, input)); + let (res, transcribe_err) = dbg!(transcriber::transcribe(&rule.rhs, &bindings)); + (res, bindings_err.or(transcribe_err)) } /// The actual algorithm for expansion is not too hard, but is pretty tricky. @@ -111,7 +122,7 @@ mod tests { } fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) { - assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation), Err(err)); + assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation).1, Some(err)); } fn format_macro(macro_body: &str) -> String { @@ -138,7 +149,7 @@ mod tests { fn expand_first( rules: &crate::MacroRules, invocation: &str, - ) -> Result { + ) -> ExpandResult { let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); let macro_invocation = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 49c53183a..f9d4952c6 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -11,6 +11,7 @@ use crate::{ use ra_parser::{FragmentKind::*, TreeSink}; use ra_syntax::{SmolStr, SyntaxKind}; use tt::buffer::{Cursor, TokenBuffer}; +use super::ExpandResult; impl Bindings { fn push_optional(&mut self, name: &SmolStr) { @@ -64,19 +65,19 @@ macro_rules! bail { }; } -pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result { +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult { assert!(pattern.delimiter == None); let mut res = Bindings::default(); let mut src = TtIter::new(src); - match_subtree(&mut res, pattern, &mut src)?; + let mut err = match_subtree(&mut res, pattern, &mut src).err(); - if src.len() > 0 { - bail!("leftover tokens"); + if src.len() > 0 && err.is_none() { + err = Some(err!("leftover tokens")); } - Ok(res) + (res, err) } fn match_subtree( diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index 7662020f3..c53c2d35e 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -3,6 +3,7 @@ use ra_syntax::SmolStr; +use super::ExpandResult; use crate::{ mbe_expander::{Binding, Bindings, Fragment}, parser::{parse_template, Op, RepeatKind, Separator}, @@ -49,10 +50,7 @@ impl Bindings { } } -pub(super) fn transcribe( - template: &tt::Subtree, - bindings: &Bindings, -) -> Result { +pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult { assert!(template.delimiter == None); let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; expand_subtree(&mut ctx, template) @@ -75,35 +73,46 @@ struct ExpandCtx<'a> { nesting: Vec, } -fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result { +fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> ExpandResult { let mut buf: Vec = Vec::new(); + let mut err = None; for op in parse_template(template) { - match op? { + let op = match op { + Ok(op) => op, + Err(e) => { + err = Some(e); + break; + } + }; + match op { Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), Op::TokenTree(tt::TokenTree::Subtree(tt)) => { - let tt = expand_subtree(ctx, tt)?; + let (tt, e) = expand_subtree(ctx, tt); + err = err.or(e); buf.push(tt.into()); } Op::Var { name, kind: _ } => { - let fragment = expand_var(ctx, name)?; + let (fragment, e) = expand_var(ctx, name); + err = err.or(e); push_fragment(&mut buf, fragment); } Op::Repeat { subtree, kind, separator } => { - let fragment = expand_repeat(ctx, subtree, kind, separator)?; + let (fragment, e) = expand_repeat(ctx, subtree, kind, separator); + err = err.or(e); push_fragment(&mut buf, fragment) } } } - Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) + (tt::Subtree { delimiter: template.delimiter, token_trees: buf }, err) } -fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result { - let res = if v == "crate" { +fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { + if v == "crate" { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. let tt = tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) .into(); - Fragment::Tokens(tt) + (Fragment::Tokens(tt), None) } else if !ctx.bindings.contains(v) { // Note that it is possible to have a `$var` inside a macro which is not bound. // For example: @@ -132,11 +141,13 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result ], } .into(); - Fragment::Tokens(tt) + (Fragment::Tokens(tt), None) } else { - ctx.bindings.get(&v, &mut ctx.nesting)?.clone() - }; - Ok(res) + ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( + |e| (Fragment::Tokens(tt::TokenTree::empty()), Some(e)), + |b| (b.clone(), None), + ) + } } fn expand_repeat( @@ -144,17 +155,17 @@ fn expand_repeat( template: &tt::Subtree, kind: RepeatKind, separator: Option, -) -> Result { +) -> ExpandResult { let mut buf: Vec = Vec::new(); ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); // Dirty hack to make macro-expansion terminate. - // This should be replaced by a propper macro-by-example implementation + // This should be replaced by a proper macro-by-example implementation let limit = 65536; let mut has_seps = 0; let mut counter = 0; loop { - let res = expand_subtree(ctx, template); + let (mut t, e) = expand_subtree(ctx, template); let nesting_state = ctx.nesting.last_mut().unwrap(); if nesting_state.at_end || !nesting_state.hit { break; @@ -172,10 +183,10 @@ fn expand_repeat( break; } - let mut t = match res { - Ok(t) => t, - Err(_) => continue, - }; + if e.is_some() { + continue; + } + t.delimiter = None; push_subtree(&mut buf, t); @@ -209,14 +220,14 @@ fn expand_repeat( buf.pop(); } - if RepeatKind::OneOrMore == kind && counter == 0 { - return Err(ExpandError::UnexpectedToken); - } - // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); - Ok(Fragment::Tokens(tt)) + + if RepeatKind::OneOrMore == kind && counter == 0 { + return (Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken)); + } + (Fragment::Tokens(tt), None) } fn push_fragment(buf: &mut Vec, fragment: Fragment) { diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 6d5d1e9e6..4d3140fa9 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -1430,7 +1430,8 @@ impl MacroFixture { let (invocation_tt, _) = ast_to_token_tree(¯o_invocation.token_tree().unwrap()).unwrap(); - self.rules.expand(&invocation_tt) + let (tt, err) = self.rules.expand(&invocation_tt); + err.map(Err).unwrap_or(Ok(tt)) } fn assert_expand_err(&self, invocation: &str, err: &ExpandError) { -- cgit v1.2.3