From b973158aeb337041d4e1434cf5d8c609a0b02bef Mon Sep 17 00:00:00 2001 From: Florian Diebold Date: Fri, 13 Mar 2020 13:03:31 +0100 Subject: Make MBE expansion more resilient (WIP) --- crates/ra_mbe/src/mbe_expander/matcher.rs | 11 +++-- crates/ra_mbe/src/mbe_expander/transcriber.rs | 69 ++++++++++++++++----------- 2 files changed, 46 insertions(+), 34 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 49c53183a..f9d4952c6 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -11,6 +11,7 @@ use crate::{ use ra_parser::{FragmentKind::*, TreeSink}; use ra_syntax::{SmolStr, SyntaxKind}; use tt::buffer::{Cursor, TokenBuffer}; +use super::ExpandResult; impl Bindings { fn push_optional(&mut self, name: &SmolStr) { @@ -64,19 +65,19 @@ macro_rules! bail { }; } -pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result { +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult { assert!(pattern.delimiter == None); let mut res = Bindings::default(); let mut src = TtIter::new(src); - match_subtree(&mut res, pattern, &mut src)?; + let mut err = match_subtree(&mut res, pattern, &mut src).err(); - if src.len() > 0 { - bail!("leftover tokens"); + if src.len() > 0 && err.is_none() { + err = Some(err!("leftover tokens")); } - Ok(res) + (res, err) } fn match_subtree( diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index 7662020f3..c53c2d35e 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -3,6 +3,7 @@ use ra_syntax::SmolStr; +use super::ExpandResult; use crate::{ mbe_expander::{Binding, Bindings, Fragment}, parser::{parse_template, Op, RepeatKind, Separator}, @@ -49,10 +50,7 @@ impl Bindings { } } -pub(super) fn transcribe( - template: &tt::Subtree, - bindings: &Bindings, -) -> Result { +pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult { assert!(template.delimiter == None); let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; expand_subtree(&mut ctx, template) @@ -75,35 +73,46 @@ struct ExpandCtx<'a> { nesting: Vec, } -fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result { +fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> ExpandResult { let mut buf: Vec = Vec::new(); + let mut err = None; for op in parse_template(template) { - match op? { + let op = match op { + Ok(op) => op, + Err(e) => { + err = Some(e); + break; + } + }; + match op { Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), Op::TokenTree(tt::TokenTree::Subtree(tt)) => { - let tt = expand_subtree(ctx, tt)?; + let (tt, e) = expand_subtree(ctx, tt); + err = err.or(e); buf.push(tt.into()); } Op::Var { name, kind: _ } => { - let fragment = expand_var(ctx, name)?; + let (fragment, e) = expand_var(ctx, name); + err = err.or(e); push_fragment(&mut buf, fragment); } Op::Repeat { subtree, kind, separator } => { - let fragment = expand_repeat(ctx, subtree, kind, separator)?; + let (fragment, e) = expand_repeat(ctx, subtree, kind, separator); + err = err.or(e); push_fragment(&mut buf, fragment) } } } - Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) + (tt::Subtree { delimiter: template.delimiter, token_trees: buf }, err) } -fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result { - let res = if v == "crate" { +fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { + if v == "crate" { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. let tt = tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) .into(); - Fragment::Tokens(tt) + (Fragment::Tokens(tt), None) } else if !ctx.bindings.contains(v) { // Note that it is possible to have a `$var` inside a macro which is not bound. // For example: @@ -132,11 +141,13 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result ], } .into(); - Fragment::Tokens(tt) + (Fragment::Tokens(tt), None) } else { - ctx.bindings.get(&v, &mut ctx.nesting)?.clone() - }; - Ok(res) + ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( + |e| (Fragment::Tokens(tt::TokenTree::empty()), Some(e)), + |b| (b.clone(), None), + ) + } } fn expand_repeat( @@ -144,17 +155,17 @@ fn expand_repeat( template: &tt::Subtree, kind: RepeatKind, separator: Option, -) -> Result { +) -> ExpandResult { let mut buf: Vec = Vec::new(); ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); // Dirty hack to make macro-expansion terminate. - // This should be replaced by a propper macro-by-example implementation + // This should be replaced by a proper macro-by-example implementation let limit = 65536; let mut has_seps = 0; let mut counter = 0; loop { - let res = expand_subtree(ctx, template); + let (mut t, e) = expand_subtree(ctx, template); let nesting_state = ctx.nesting.last_mut().unwrap(); if nesting_state.at_end || !nesting_state.hit { break; @@ -172,10 +183,10 @@ fn expand_repeat( break; } - let mut t = match res { - Ok(t) => t, - Err(_) => continue, - }; + if e.is_some() { + continue; + } + t.delimiter = None; push_subtree(&mut buf, t); @@ -209,14 +220,14 @@ fn expand_repeat( buf.pop(); } - if RepeatKind::OneOrMore == kind && counter == 0 { - return Err(ExpandError::UnexpectedToken); - } - // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); - Ok(Fragment::Tokens(tt)) + + if RepeatKind::OneOrMore == kind && counter == 0 { + return (Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken)); + } + (Fragment::Tokens(tt), None) } fn push_fragment(buf: &mut Vec, fragment: Fragment) { -- cgit v1.2.3 From 6305d094ac61ed6e437537b93f4e587b415678c9 Mon Sep 17 00:00:00 2001 From: Florian Diebold Date: Fri, 13 Mar 2020 15:18:17 +0100 Subject: Attempt to implement ranking of rules when none matches perfectly (wip) --- crates/ra_mbe/src/mbe_expander/matcher.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index f9d4952c6..52f1ac252 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -65,7 +65,7 @@ macro_rules! bail { }; } -pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult { +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult<(Bindings, usize)> { assert!(pattern.delimiter == None); let mut res = Bindings::default(); @@ -77,7 +77,7 @@ pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult Date: Sat, 14 Mar 2020 20:24:18 +0100 Subject: wip --- crates/ra_mbe/src/mbe_expander/matcher.rs | 172 +++++++++++++++++++----------- 1 file changed, 108 insertions(+), 64 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 52f1ac252..ae65fb69a 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -8,10 +8,10 @@ use crate::{ ExpandError, }; +use super::ExpandResult; use ra_parser::{FragmentKind::*, TreeSink}; use ra_syntax::{SmolStr, SyntaxKind}; use tt::buffer::{Cursor, TokenBuffer}; -use super::ExpandResult; impl Bindings { fn push_optional(&mut self, name: &SmolStr) { @@ -59,36 +59,50 @@ macro_rules! err { }; } -macro_rules! bail { - ($($tt:tt)*) => { - return Err(err!($($tt)*)) - }; +#[derive(Debug, Default)] +pub(super) struct Match { + pub bindings: Bindings, + pub unmatched_tokens: usize, + pub unmatched_patterns: usize, } -pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult<(Bindings, usize)> { +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult { assert!(pattern.delimiter == None); - let mut res = Bindings::default(); + let mut res = Match::default(); let mut src = TtIter::new(src); let mut err = match_subtree(&mut res, pattern, &mut src).err(); + res.unmatched_tokens += src.len(); if src.len() > 0 && err.is_none() { err = Some(err!("leftover tokens")); } - ((res, src.len()), err) + (res, err) } fn match_subtree( - bindings: &mut Bindings, + res: &mut Match, pattern: &tt::Subtree, src: &mut TtIter, ) -> Result<(), ExpandError> { + let mut result = Ok(()); for op in parse_pattern(pattern) { + if result.is_err() { + // We're just going through the patterns to count how many we missed + res.unmatched_patterns += 1; + continue; + } match op? { Op::TokenTree(tt::TokenTree::Leaf(lhs)) => { - let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?; + let rhs = match src.expect_leaf() { + Ok(l) => l, + Err(()) => { + result = Err(err!("expected leaf: `{}`", lhs)); + continue; + } + }; match (lhs, rhs) { ( tt::Leaf::Punct(tt::Punct { char: lhs, .. }), @@ -102,35 +116,54 @@ fn match_subtree( tt::Leaf::Literal(tt::Literal { text: lhs, .. }), tt::Leaf::Literal(tt::Literal { text: rhs, .. }), ) if lhs == rhs => (), - _ => return Err(ExpandError::UnexpectedToken), + _ => { + result = Err(ExpandError::UnexpectedToken); + } } } Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { - let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; + let rhs = match src.expect_subtree() { + Ok(s) => s, + Err(()) => { + result = Err(err!("expected subtree")); + continue; + } + }; if lhs.delimiter_kind() != rhs.delimiter_kind() { - bail!("mismatched delimiter") + result = Err(err!("mismatched delimiter")); + continue; } let mut src = TtIter::new(rhs); - match_subtree(bindings, lhs, &mut src)?; - if src.len() > 0 { - bail!("leftover tokens"); + result = match_subtree(res, lhs, &mut src); + res.unmatched_tokens += src.len(); + if src.len() > 0 && result.is_ok() { + result = Err(err!("leftover tokens")); } } Op::Var { name, kind } => { - let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; - match match_meta_var(kind.as_str(), src)? { + let kind = match kind { + Some(k) => k, + None => { + result = Err(ExpandError::UnexpectedToken); + continue; + } + }; + let (matched, match_err) = match_meta_var(kind.as_str(), src); + match matched { Some(fragment) => { - bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); + res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); } - None => bindings.push_optional(name), + None if match_err.is_none() => res.bindings.push_optional(name), + _ => {} } + result = match_err.map_or(Ok(()), Err); } Op::Repeat { subtree, kind, separator } => { - match_repeat(bindings, subtree, kind, separator, src)? + result = match_repeat(res, subtree, kind, separator, src); } } } - Ok(()) + result } impl<'a> TtIter<'a> { @@ -222,7 +255,7 @@ impl<'a> TtIter<'a> { pub(crate) fn expect_fragment( &mut self, fragment_kind: ra_parser::FragmentKind, - ) -> Result { + ) -> ExpandResult { pub(crate) struct OffsetTokenSink<'a> { pub(crate) cursor: Cursor<'a>, pub(crate) error: bool, @@ -247,45 +280,47 @@ impl<'a> TtIter<'a> { ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind); + let mut err = None; if !sink.cursor.is_root() || sink.error { - // FIXME better recovery in this case would help completion inside macros immensely - return Err(()); + err = Some(err!("expected {:?}", fragment_kind)); } let mut curr = buffer.begin(); let mut res = vec![]; - while curr != sink.cursor { - if let Some(token) = curr.token_tree() { - res.push(token); + if sink.cursor.is_root() { + while curr != sink.cursor { + if let Some(token) = curr.token_tree() { + res.push(token); + } + curr = curr.bump(); } - curr = curr.bump(); } self.inner = self.inner.as_slice()[res.len()..].iter(); - match res.len() { - 0 => Err(()), - 1 => Ok(res[0].clone()), - _ => Ok(tt::TokenTree::Subtree(tt::Subtree { + let res = match res.len() { + 1 => res[0].clone(), + _ => tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees: res.into_iter().cloned().collect(), - })), - } + }), + }; + (res, err) } pub(crate) fn eat_vis(&mut self) -> Option { let mut fork = self.clone(); match fork.expect_fragment(Visibility) { - Ok(tt) => { + (tt, None) => { *self = fork; Some(tt) } - Err(()) => None, + (_, Some(_)) => None, } } } pub(super) fn match_repeat( - bindings: &mut Bindings, + res: &mut Match, pattern: &tt::Subtree, kind: RepeatKind, separator: Option, @@ -305,17 +340,23 @@ pub(super) fn match_repeat( } } - let mut nested = Bindings::default(); + let mut nested = Match::default(); match match_subtree(&mut nested, pattern, &mut fork) { Ok(()) => { limit -= 1; if limit == 0 { - log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator); + log::warn!( + "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", + pattern, + src, + kind, + separator + ); break; } *src = fork; - bindings.push_nested(counter, nested)?; + res.bindings.push_nested(counter, nested.bindings)?; counter += 1; if counter == 1 { if let RepeatKind::ZeroOrOne = kind { @@ -334,7 +375,7 @@ pub(super) fn match_repeat( let mut vars = Vec::new(); collect_vars(&mut vars, pattern)?; for var in vars { - bindings.push_empty(&var) + res.bindings.push_empty(&var) } } _ => (), @@ -342,7 +383,7 @@ pub(super) fn match_repeat( Ok(()) } -fn match_meta_var(kind: &str, input: &mut TtIter) -> Result, ExpandError> { +fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult> { let fragment = match kind { "path" => Path, "expr" => Expr, @@ -353,34 +394,33 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> Result, Ex "meta" => MetaItem, "item" => Item, _ => { - let tt = match kind { - "ident" => { - let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone(); - tt::Leaf::from(ident).into() - } - "tt" => input.expect_tt().map_err(|()| err!())?.clone(), - "lifetime" => { - let ident = input.expect_lifetime().map_err(|()| err!())?; - tt::Leaf::Ident(ident.clone()).into() - } - "literal" => { - let literal = input.expect_literal().map_err(|()| err!())?.clone(); - tt::Leaf::from(literal).into() - } + let tt_result = match kind { + "ident" => input + .expect_ident() + .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) + .map_err(|()| err!("expected ident")), + "tt" => input.expect_tt().map(Some).map_err(|()| err!()), + "lifetime" => input + .expect_lifetime() + .map(|ident| Some(tt::Leaf::Ident(ident.clone()).into())) + .map_err(|()| err!("expected lifetime")), + "literal" => input + .expect_literal() + .map(|literal| Some(tt::Leaf::from(literal.clone()).into())) + .map_err(|()| err!()), // `vis` is optional "vis" => match input.eat_vis() { - Some(vis) => vis, - None => return Ok(None), + Some(vis) => Ok(Some(vis)), + None => Ok(None), }, - _ => return Err(ExpandError::UnexpectedToken), + _ => Err(ExpandError::UnexpectedToken), }; - return Ok(Some(Fragment::Tokens(tt))); + return to_expand_result(tt_result.map(|it| it.map(Fragment::Tokens))); } }; - let tt = - input.expect_fragment(fragment).map_err(|()| err!("fragment did not parse as {}", kind))?; + let (tt, err) = input.expect_fragment(fragment); let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; - Ok(Some(fragment)) + (Some(fragment), err) } fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), ExpandError> { @@ -394,3 +434,7 @@ fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), Exp } Ok(()) } + +fn to_expand_result(result: Result) -> ExpandResult { + result.map_or_else(|e| (Default::default(), Some(e)), |it| (it, None)) +} -- cgit v1.2.3 From f3c6a2e3dbe477a7e0ac714a5bdbda6e8838fcfa Mon Sep 17 00:00:00 2001 From: Florian Diebold Date: Sun, 15 Mar 2020 15:15:56 +0100 Subject: Fix remaining test failure --- crates/ra_mbe/src/mbe_expander/matcher.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index ae65fb69a..f40d55d4e 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -255,7 +255,7 @@ impl<'a> TtIter<'a> { pub(crate) fn expect_fragment( &mut self, fragment_kind: ra_parser::FragmentKind, - ) -> ExpandResult { + ) -> ExpandResult> { pub(crate) struct OffsetTokenSink<'a> { pub(crate) cursor: Cursor<'a>, pub(crate) error: bool, @@ -297,12 +297,16 @@ impl<'a> TtIter<'a> { } } self.inner = self.inner.as_slice()[res.len()..].iter(); + if res.len() == 0 && err.is_none() { + err = Some(err!("no tokens consumed")); + } let res = match res.len() { - 1 => res[0].clone(), - _ => tt::TokenTree::Subtree(tt::Subtree { + 1 => Some(res[0].clone()), + 0 => None, + _ => Some(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees: res.into_iter().cloned().collect(), - }), + })), }; (res, err) } @@ -312,7 +316,7 @@ impl<'a> TtIter<'a> { match fork.expect_fragment(Visibility) { (tt, None) => { *self = fork; - Some(tt) + tt } (_, Some(_)) => None, } @@ -419,8 +423,8 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult, pattern: &tt::Subtree) -> Result<(), ExpandError> { -- cgit v1.2.3 From d655749aaeb31461f9af923bbf0b36d219cff343 Mon Sep 17 00:00:00 2001 From: Florian Diebold Date: Mon, 16 Mar 2020 12:22:10 +0100 Subject: Turn ExpandResult into struct --- crates/ra_mbe/src/mbe_expander/matcher.rs | 21 ++++++++------------- crates/ra_mbe/src/mbe_expander/transcriber.rs | 22 +++++++++++----------- 2 files changed, 19 insertions(+), 24 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index f40d55d4e..22fb5d74c 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -79,7 +79,7 @@ pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult { res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); @@ -308,17 +308,17 @@ impl<'a> TtIter<'a> { token_trees: res.into_iter().cloned().collect(), })), }; - (res, err) + ExpandResult(res, err) } pub(crate) fn eat_vis(&mut self) -> Option { let mut fork = self.clone(); match fork.expect_fragment(Visibility) { - (tt, None) => { + ExpandResult(tt, None) => { *self = fork; tt } - (_, Some(_)) => None, + ExpandResult(_, Some(_)) => None, } } } @@ -419,12 +419,11 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult Err(ExpandError::UnexpectedToken), }; - return to_expand_result(tt_result.map(|it| it.map(Fragment::Tokens))); + return tt_result.map(|it| it.map(Fragment::Tokens)).into(); } }; - let (tt, err) = input.expect_fragment(fragment); - let fragment = if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }; - (fragment, err) + let result = input.expect_fragment(fragment); + result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) } fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), ExpandError> { @@ -438,7 +437,3 @@ fn collect_vars(buf: &mut Vec, pattern: &tt::Subtree) -> Result<(), Exp } Ok(()) } - -fn to_expand_result(result: Result) -> ExpandResult { - result.map_or_else(|e| (Default::default(), Some(e)), |it| (it, None)) -} diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index c53c2d35e..4b173edd3 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -87,23 +87,23 @@ fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> ExpandResult buf.push(tt.clone()), Op::TokenTree(tt::TokenTree::Subtree(tt)) => { - let (tt, e) = expand_subtree(ctx, tt); + let ExpandResult(tt, e) = expand_subtree(ctx, tt); err = err.or(e); buf.push(tt.into()); } Op::Var { name, kind: _ } => { - let (fragment, e) = expand_var(ctx, name); + let ExpandResult(fragment, e) = expand_var(ctx, name); err = err.or(e); push_fragment(&mut buf, fragment); } Op::Repeat { subtree, kind, separator } => { - let (fragment, e) = expand_repeat(ctx, subtree, kind, separator); + let ExpandResult(fragment, e) = expand_repeat(ctx, subtree, kind, separator); err = err.or(e); push_fragment(&mut buf, fragment) } } } - (tt::Subtree { delimiter: template.delimiter, token_trees: buf }, err) + ExpandResult(tt::Subtree { delimiter: template.delimiter, token_trees: buf }, err) } fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { @@ -112,7 +112,7 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { let tt = tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) .into(); - (Fragment::Tokens(tt), None) + ExpandResult::ok(Fragment::Tokens(tt)) } else if !ctx.bindings.contains(v) { // Note that it is possible to have a `$var` inside a macro which is not bound. // For example: @@ -141,11 +141,11 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult { ], } .into(); - (Fragment::Tokens(tt), None) + ExpandResult::ok(Fragment::Tokens(tt)) } else { ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( - |e| (Fragment::Tokens(tt::TokenTree::empty()), Some(e)), - |b| (b.clone(), None), + |e| ExpandResult(Fragment::Tokens(tt::TokenTree::empty()), Some(e)), + |b| ExpandResult::ok(b.clone()), ) } } @@ -165,7 +165,7 @@ fn expand_repeat( let mut counter = 0; loop { - let (mut t, e) = expand_subtree(ctx, template); + let ExpandResult(mut t, e) = expand_subtree(ctx, template); let nesting_state = ctx.nesting.last_mut().unwrap(); if nesting_state.at_end || !nesting_state.hit { break; @@ -225,9 +225,9 @@ fn expand_repeat( let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); if RepeatKind::OneOrMore == kind && counter == 0 { - return (Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken)); + return ExpandResult(Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken)); } - (Fragment::Tokens(tt), None) + ExpandResult::ok(Fragment::Tokens(tt)) } fn push_fragment(buf: &mut Vec, fragment: Fragment) { -- cgit v1.2.3 From 32dce75747271236be0ca8762b338c317bd48204 Mon Sep 17 00:00:00 2001 From: Florian Diebold Date: Mon, 16 Mar 2020 18:38:10 +0100 Subject: Some more refactoring --- crates/ra_mbe/src/mbe_expander/matcher.rs | 112 +++++++++++++++++------------- 1 file changed, 64 insertions(+), 48 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander') diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 22fb5d74c..2579382da 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs @@ -62,24 +62,41 @@ macro_rules! err { #[derive(Debug, Default)] pub(super) struct Match { pub bindings: Bindings, - pub unmatched_tokens: usize, - pub unmatched_patterns: usize, + /// We currently just keep the first error and count the rest to compare matches. + pub err: Option, + pub err_count: usize, + /// How many top-level token trees were left to match. + pub unmatched_tts: usize, } -pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult { +impl Match { + pub fn add_err(&mut self, err: ExpandError) { + let prev_err = self.err.take(); + self.err = prev_err.or(Some(err)); + self.err_count += 1; + } +} + +// General note: These functions have two channels to return errors, a `Result` +// return value and the `&mut Match`. The returned Result is for pattern parsing +// errors; if a branch of the macro definition doesn't parse, it doesn't make +// sense to try using it. Matching errors are added to the `Match`. It might +// make sense to make pattern parsing a separate step? + +pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result { assert!(pattern.delimiter == None); let mut res = Match::default(); let mut src = TtIter::new(src); - let mut err = match_subtree(&mut res, pattern, &mut src).err(); + match_subtree(&mut res, pattern, &mut src)?; - res.unmatched_tokens += src.len(); - if src.len() > 0 && err.is_none() { - err = Some(err!("leftover tokens")); + if src.len() > 0 { + res.unmatched_tts += src.len(); + res.add_err(err!("leftover tokens")); } - ExpandResult(res, err) + Ok(res) } fn match_subtree( @@ -87,19 +104,13 @@ fn match_subtree( pattern: &tt::Subtree, src: &mut TtIter, ) -> Result<(), ExpandError> { - let mut result = Ok(()); for op in parse_pattern(pattern) { - if result.is_err() { - // We're just going through the patterns to count how many we missed - res.unmatched_patterns += 1; - continue; - } match op? { Op::TokenTree(tt::TokenTree::Leaf(lhs)) => { let rhs = match src.expect_leaf() { Ok(l) => l, Err(()) => { - result = Err(err!("expected leaf: `{}`", lhs)); + res.add_err(err!("expected leaf: `{}`", lhs)); continue; } }; @@ -117,7 +128,7 @@ fn match_subtree( tt::Leaf::Literal(tt::Literal { text: rhs, .. }), ) if lhs == rhs => (), _ => { - result = Err(ExpandError::UnexpectedToken); + res.add_err(ExpandError::UnexpectedToken); } } } @@ -125,26 +136,25 @@ fn match_subtree( let rhs = match src.expect_subtree() { Ok(s) => s, Err(()) => { - result = Err(err!("expected subtree")); + res.add_err(err!("expected subtree")); continue; } }; if lhs.delimiter_kind() != rhs.delimiter_kind() { - result = Err(err!("mismatched delimiter")); + res.add_err(err!("mismatched delimiter")); continue; } let mut src = TtIter::new(rhs); - result = match_subtree(res, lhs, &mut src); - res.unmatched_tokens += src.len(); - if src.len() > 0 && result.is_ok() { - result = Err(err!("leftover tokens")); + match_subtree(res, lhs, &mut src)?; + if src.len() > 0 { + res.add_err(err!("leftover tokens")); } } Op::Var { name, kind } => { let kind = match kind { Some(k) => k, None => { - result = Err(ExpandError::UnexpectedToken); + res.add_err(ExpandError::UnexpectedToken); continue; } }; @@ -156,14 +166,16 @@ fn match_subtree( None if match_err.is_none() => res.bindings.push_optional(name), _ => {} } - result = match_err.map_or(Ok(()), Err); + if let Some(err) = match_err { + res.add_err(err); + } } Op::Repeat { subtree, kind, separator } => { - result = match_repeat(res, subtree, kind, separator, src); + match_repeat(res, subtree, kind, separator, src)?; } } } - result + Ok(()) } impl<'a> TtIter<'a> { @@ -345,35 +357,39 @@ pub(super) fn match_repeat( } let mut nested = Match::default(); - match match_subtree(&mut nested, pattern, &mut fork) { - Ok(()) => { - limit -= 1; - if limit == 0 { - log::warn!( - "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", - pattern, - src, - kind, - separator - ); - break; - } - *src = fork; + match_subtree(&mut nested, pattern, &mut fork)?; + if nested.err.is_none() { + limit -= 1; + if limit == 0 { + log::warn!( + "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", + pattern, + src, + kind, + separator + ); + break; + } + *src = fork; - res.bindings.push_nested(counter, nested.bindings)?; - counter += 1; - if counter == 1 { - if let RepeatKind::ZeroOrOne = kind { - break; - } + if let Err(err) = res.bindings.push_nested(counter, nested.bindings) { + res.add_err(err); + } + counter += 1; + if counter == 1 { + if let RepeatKind::ZeroOrOne = kind { + break; } } - Err(_) => break, + } else { + break; } } match (kind, counter) { - (RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken), + (RepeatKind::OneOrMore, 0) => { + res.add_err(ExpandError::UnexpectedToken); + } (_, 0) => { // Collect all empty variables in subtrees let mut vars = Vec::new(); -- cgit v1.2.3