From 4c7d8cbfbfd2d93008c1266eb45c2671bd49b60e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 29 Jan 2021 20:23:38 +0800 Subject: Rename mbe_expander for consistency --- crates/mbe/src/expander.rs | 182 +++++++++++ crates/mbe/src/expander/matcher.rs | 502 +++++++++++++++++++++++++++++ crates/mbe/src/expander/transcriber.rs | 248 ++++++++++++++ crates/mbe/src/lib.rs | 6 +- crates/mbe/src/mbe_expander.rs | 182 ----------- crates/mbe/src/mbe_expander/matcher.rs | 502 ----------------------------- crates/mbe/src/mbe_expander/transcriber.rs | 248 -------------- 7 files changed, 935 insertions(+), 935 deletions(-) create mode 100644 crates/mbe/src/expander.rs create mode 100644 crates/mbe/src/expander/matcher.rs create mode 100644 crates/mbe/src/expander/transcriber.rs delete mode 100644 crates/mbe/src/mbe_expander.rs delete mode 100644 crates/mbe/src/mbe_expander/matcher.rs delete mode 100644 crates/mbe/src/mbe_expander/transcriber.rs diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs new file mode 100644 index 000000000..802c8fb0f --- /dev/null +++ b/crates/mbe/src/expander.rs @@ -0,0 +1,182 @@ +//! This module takes a (parsed) definition of `macro_rules` invocation, a +//! `tt::TokenTree` representing an argument of macro invocation, and produces a +//! `tt::TokenTree` for the result of the expansion. + +mod matcher; +mod transcriber; + +use rustc_hash::FxHashMap; +use syntax::SmolStr; + +use crate::{ExpandError, ExpandResult}; + +pub(crate) fn expand_rules( + rules: &[crate::Rule], + input: &tt::Subtree, +) -> ExpandResult { + let mut match_: Option<(matcher::Match, &crate::Rule)> = None; + for rule in rules { + let new_match = match matcher::match_(&rule.lhs, input) { + Ok(m) => m, + Err(_e) => { + // error in pattern parsing + continue; + } + }; + if new_match.err.is_none() { + // If we find a rule that applies without errors, we're done. + // Unconditionally returning the transcription here makes the + // `test_repeat_bad_var` test fail. + let ExpandResult { value, err: transcribe_err } = + transcriber::transcribe(&rule.rhs, &new_match.bindings); + if transcribe_err.is_none() { + return ExpandResult::ok(value); + } + } + // Use the rule if we matched more tokens, or had fewer errors + if let Some((prev_match, _)) = &match_ { + if (new_match.unmatched_tts, new_match.err_count) + < (prev_match.unmatched_tts, prev_match.err_count) + { + match_ = Some((new_match, rule)); + } + } else { + match_ = Some((new_match, rule)); + } + } + if let Some((match_, rule)) = match_ { + // if we got here, there was no match without errors + let ExpandResult { value, err: transcribe_err } = + transcriber::transcribe(&rule.rhs, &match_.bindings); + ExpandResult { value, err: match_.err.or(transcribe_err) } + } else { + ExpandResult::only_err(ExpandError::NoMatchingRule) + } +} + +/// The actual algorithm for expansion is not too hard, but is pretty tricky. +/// `Bindings` structure is the key to understanding what we are doing here. +/// +/// On the high level, it stores mapping from meta variables to the bits of +/// syntax it should be substituted with. For example, if `$e:expr` is matched +/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`. +/// +/// The tricky bit is dealing with repetitions (`$()*`). Consider this example: +/// +/// ```not_rust +/// macro_rules! foo { +/// ($($ i:ident $($ e:expr),*);*) => { +/// $(fn $ i() { $($ e);*; })* +/// } +/// } +/// foo! { foo 1,2,3; bar 4,5,6 } +/// ``` +/// +/// Here, the `$i` meta variable is matched first with `foo` and then with +/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`. +/// +/// To represent such "multi-mappings", we use a recursive structures: we map +/// variables not to values, but to *lists* of values or other lists (that is, +/// to the trees). +/// +/// For the above example, the bindings would store +/// +/// ```not_rust +/// i -> [foo, bar] +/// e -> [[1, 2, 3], [4, 5, 6]] +/// ``` +/// +/// We construct `Bindings` in the `match_lhs`. The interesting case is +/// `TokenTree::Repeat`, where we use `push_nested` to create the desired +/// nesting structure. +/// +/// The other side of the puzzle is `expand_subtree`, where we use the bindings +/// to substitute meta variables in the output template. When expanding, we +/// maintain a `nesting` stack of indices which tells us which occurrence from +/// the `Bindings` we should take. We push to the stack when we enter a +/// repetition. +/// +/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to +/// `tt::TokenTree`, where the index to select a particular `TokenTree` among +/// many is not a plain `usize`, but an `&[usize]`. +#[derive(Debug, Default)] +struct Bindings { + inner: FxHashMap, +} + +#[derive(Debug)] +enum Binding { + Fragment(Fragment), + Nested(Vec), + Empty, +} + +#[derive(Debug, Clone)] +enum Fragment { + /// token fragments are just copy-pasted into the output + Tokens(tt::TokenTree), + /// Ast fragments are inserted with fake delimiters, so as to make things + /// like `$i * 2` where `$i = 1 + 1` work as expectd. + Ast(tt::TokenTree), +} + +#[cfg(test)] +mod tests { + use syntax::{ast, AstNode}; + + use super::*; + use crate::ast_to_token_tree; + + #[test] + fn test_expand_rule() { + assert_err( + "($($i:ident);*) => ($i)", + "foo!{a}", + ExpandError::BindingError(String::from( + "expected simple binding, found nested binding `i`", + )), + ); + + // FIXME: + // Add an err test case for ($($i:ident)) => ($()) + } + + fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) { + assert_eq!( + expand_first(&create_rules(&format_macro(macro_body)), invocation).err, + Some(err) + ); + } + + fn format_macro(macro_body: &str) -> String { + format!( + " + macro_rules! foo {{ + {} + }} +", + macro_body + ) + } + + fn create_rules(macro_definition: &str) -> crate::MacroRules { + let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap(); + let macro_definition = + source_file.syntax().descendants().find_map(ast::MacroRules::cast).unwrap(); + + let (definition_tt, _) = + ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); + crate::MacroRules::parse(&definition_tt).unwrap() + } + + fn expand_first(rules: &crate::MacroRules, invocation: &str) -> ExpandResult { + let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); + let macro_invocation = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); + + let (invocation_tt, _) = + ast_to_token_tree(¯o_invocation.token_tree().unwrap()).unwrap(); + + expand_rules(&rules.rules, &invocation_tt) + } +} diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs new file mode 100644 index 000000000..5b5845850 --- /dev/null +++ b/crates/mbe/src/expander/matcher.rs @@ -0,0 +1,502 @@ +//! FIXME: write short doc here + +use crate::{ + expander::{Binding, Bindings, Fragment}, + parser::{Op, RepeatKind, Separator}, + subtree_source::SubtreeTokenSource, + tt_iter::TtIter, + ExpandError, MetaTemplate, +}; + +use super::ExpandResult; +use parser::{FragmentKind::*, TreeSink}; +use syntax::{SmolStr, SyntaxKind}; +use tt::buffer::{Cursor, TokenBuffer}; + +impl Bindings { + fn push_optional(&mut self, name: &SmolStr) { + // FIXME: Do we have a better way to represent an empty token ? + // Insert an empty subtree for empty token + let tt = tt::Subtree::default().into(); + self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); + } + + fn push_empty(&mut self, name: &SmolStr) { + self.inner.insert(name.clone(), Binding::Empty); + } + + fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> { + for (key, value) in nested.inner { + if !self.inner.contains_key(&key) { + self.inner.insert(key.clone(), Binding::Nested(Vec::new())); + } + match self.inner.get_mut(&key) { + Some(Binding::Nested(it)) => { + // insert empty nested bindings before this one + while it.len() < idx { + it.push(Binding::Nested(vec![])); + } + it.push(value); + } + _ => { + return Err(ExpandError::BindingError(format!( + "could not find binding `{}`", + key + ))); + } + } + } + Ok(()) + } +} + +macro_rules! err { + () => { + ExpandError::BindingError(format!("")) + }; + ($($tt:tt)*) => { + ExpandError::BindingError(format!($($tt)*)) + }; +} + +#[derive(Debug, Default)] +pub(super) struct Match { + pub(super) bindings: Bindings, + /// We currently just keep the first error and count the rest to compare matches. + pub(super) err: Option, + pub(super) err_count: usize, + /// How many top-level token trees were left to match. + pub(super) unmatched_tts: usize, +} + +impl Match { + pub(super) fn add_err(&mut self, err: ExpandError) { + let prev_err = self.err.take(); + self.err = prev_err.or(Some(err)); + self.err_count += 1; + } +} + +// General note: These functions have two channels to return errors, a `Result` +// return value and the `&mut Match`. The returned Result is for pattern parsing +// errors; if a branch of the macro definition doesn't parse, it doesn't make +// sense to try using it. Matching errors are added to the `Match`. It might +// make sense to make pattern parsing a separate step? + +pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Result { + assert!(pattern.delimiter == None); + + let mut res = Match::default(); + let mut src = TtIter::new(src); + + match_subtree(&mut res, pattern, &mut src)?; + + if src.len() > 0 { + res.unmatched_tts += src.len(); + res.add_err(err!("leftover tokens")); + } + + Ok(res) +} + +fn match_subtree( + res: &mut Match, + pattern: &MetaTemplate, + src: &mut TtIter, +) -> Result<(), ExpandError> { + for op in pattern.iter() { + match op.as_ref().map_err(|err| err.clone())? { + Op::Leaf(lhs) => { + let rhs = match src.expect_leaf() { + Ok(l) => l, + Err(()) => { + res.add_err(err!("expected leaf: `{}`", lhs)); + continue; + } + }; + match (lhs, rhs) { + ( + tt::Leaf::Punct(tt::Punct { char: lhs, .. }), + tt::Leaf::Punct(tt::Punct { char: rhs, .. }), + ) if lhs == rhs => (), + ( + tt::Leaf::Ident(tt::Ident { text: lhs, .. }), + tt::Leaf::Ident(tt::Ident { text: rhs, .. }), + ) if lhs == rhs => (), + ( + tt::Leaf::Literal(tt::Literal { text: lhs, .. }), + tt::Leaf::Literal(tt::Literal { text: rhs, .. }), + ) if lhs == rhs => (), + _ => { + res.add_err(ExpandError::UnexpectedToken); + } + } + } + Op::Subtree(lhs) => { + let rhs = match src.expect_subtree() { + Ok(s) => s, + Err(()) => { + res.add_err(err!("expected subtree")); + continue; + } + }; + if lhs.delimiter_kind() != rhs.delimiter_kind() { + res.add_err(err!("mismatched delimiter")); + continue; + } + let mut src = TtIter::new(rhs); + match_subtree(res, lhs, &mut src)?; + if src.len() > 0 { + res.add_err(err!("leftover tokens")); + } + } + Op::Var { name, kind, .. } => { + let kind = match kind { + Some(k) => k, + None => { + res.add_err(ExpandError::UnexpectedToken); + continue; + } + }; + let ExpandResult { value: matched, err: match_err } = + match_meta_var(kind.as_str(), src); + match matched { + Some(fragment) => { + res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); + } + None if match_err.is_none() => res.bindings.push_optional(name), + _ => {} + } + if let Some(err) = match_err { + res.add_err(err); + } + } + Op::Repeat { subtree, kind, separator } => { + match_repeat(res, subtree, *kind, separator, src)?; + } + } + } + Ok(()) +} + +impl<'a> TtIter<'a> { + fn eat_separator(&mut self, separator: &Separator) -> bool { + let mut fork = self.clone(); + let ok = match separator { + Separator::Ident(lhs) => match fork.expect_ident() { + Ok(rhs) => rhs.text == lhs.text, + _ => false, + }, + Separator::Literal(lhs) => match fork.expect_literal() { + Ok(rhs) => match rhs { + tt::Leaf::Literal(rhs) => rhs.text == lhs.text, + tt::Leaf::Ident(rhs) => rhs.text == lhs.text, + tt::Leaf::Punct(_) => false, + }, + _ => false, + }, + Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { + Ok(rhs) => rhs.char == lhs.char, + _ => false, + }), + }; + if ok { + *self = fork; + } + ok + } + + pub(crate) fn expect_tt(&mut self) -> Result { + match self.peek_n(0) { + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { + return self.expect_lifetime(); + } + _ => (), + } + + let tt = self.next().ok_or_else(|| ())?.clone(); + let punct = match tt { + tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { + punct + } + _ => return Ok(tt), + }; + + let (second, third) = match (self.peek_n(0), self.peek_n(1)) { + ( + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), + Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))), + ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)), + (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None), + _ => return Ok(tt), + }; + + match (punct.char, second, third) { + ('.', '.', Some('.')) + | ('.', '.', Some('=')) + | ('<', '<', Some('=')) + | ('>', '>', Some('=')) => { + let tt2 = self.next().unwrap().clone(); + let tt3 = self.next().unwrap().clone(); + Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into()) + } + ('-', '=', _) + | ('-', '>', _) + | (':', ':', _) + | ('!', '=', _) + | ('.', '.', _) + | ('*', '=', _) + | ('/', '=', _) + | ('&', '&', _) + | ('&', '=', _) + | ('%', '=', _) + | ('^', '=', _) + | ('+', '=', _) + | ('<', '<', _) + | ('<', '=', _) + | ('=', '=', _) + | ('=', '>', _) + | ('>', '=', _) + | ('>', '>', _) + | ('|', '=', _) + | ('|', '|', _) => { + let tt2 = self.next().unwrap().clone(); + Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into()) + } + _ => Ok(tt), + } + } + + pub(crate) fn expect_lifetime(&mut self) -> Result { + let punct = self.expect_punct()?; + if punct.char != '\'' { + return Err(()); + } + let ident = self.expect_ident()?; + + Ok(tt::Subtree { + delimiter: None, + token_trees: vec![ + tt::Leaf::Punct(*punct).into(), + tt::Leaf::Ident(ident.clone()).into(), + ], + } + .into()) + } + + pub(crate) fn expect_fragment( + &mut self, + fragment_kind: parser::FragmentKind, + ) -> ExpandResult> { + pub(crate) struct OffsetTokenSink<'a> { + pub(crate) cursor: Cursor<'a>, + pub(crate) error: bool, + } + + impl<'a> TreeSink for OffsetTokenSink<'a> { + fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { + if kind == SyntaxKind::LIFETIME_IDENT { + n_tokens = 2; + } + for _ in 0..n_tokens { + self.cursor = self.cursor.bump_subtree(); + } + } + fn start_node(&mut self, _kind: SyntaxKind) {} + fn finish_node(&mut self) {} + fn error(&mut self, _error: parser::ParseError) { + self.error = true; + } + } + + let buffer = TokenBuffer::from_tokens(&self.inner.as_slice()); + let mut src = SubtreeTokenSource::new(&buffer); + let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; + + parser::parse_fragment(&mut src, &mut sink, fragment_kind); + + let mut err = None; + if !sink.cursor.is_root() || sink.error { + err = Some(err!("expected {:?}", fragment_kind)); + } + + let mut curr = buffer.begin(); + let mut res = vec![]; + + if sink.cursor.is_root() { + while curr != sink.cursor { + if let Some(token) = curr.token_tree() { + res.push(token); + } + curr = curr.bump(); + } + } + self.inner = self.inner.as_slice()[res.len()..].iter(); + if res.len() == 0 && err.is_none() { + err = Some(err!("no tokens consumed")); + } + let res = match res.len() { + 1 => Some(res[0].cloned()), + 0 => None, + _ => Some(tt::TokenTree::Subtree(tt::Subtree { + delimiter: None, + token_trees: res.into_iter().map(|it| it.cloned()).collect(), + })), + }; + ExpandResult { value: res, err } + } + + pub(crate) fn eat_vis(&mut self) -> Option { + let mut fork = self.clone(); + match fork.expect_fragment(Visibility) { + ExpandResult { value: tt, err: None } => { + *self = fork; + tt + } + ExpandResult { value: _, err: Some(_) } => None, + } + } + + pub(crate) fn eat_char(&mut self, c: char) -> Option { + let mut fork = self.clone(); + match fork.expect_char(c) { + Ok(_) => { + let tt = self.next().cloned(); + *self = fork; + tt + } + Err(_) => None, + } + } +} + +pub(super) fn match_repeat( + res: &mut Match, + pattern: &MetaTemplate, + kind: RepeatKind, + separator: &Option, + src: &mut TtIter, +) -> Result<(), ExpandError> { + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a proper macro-by-example implementation + let mut limit = 65536; + let mut counter = 0; + + for i in 0.. { + let mut fork = src.clone(); + + if let Some(separator) = &separator { + if i != 0 && !fork.eat_separator(separator) { + break; + } + } + + let mut nested = Match::default(); + match_subtree(&mut nested, pattern, &mut fork)?; + if nested.err.is_none() { + limit -= 1; + if limit == 0 { + log::warn!( + "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", + pattern, + src, + kind, + separator + ); + break; + } + *src = fork; + + if let Err(err) = res.bindings.push_nested(counter, nested.bindings) { + res.add_err(err); + } + counter += 1; + if counter == 1 { + if let RepeatKind::ZeroOrOne = kind { + break; + } + } + } else { + break; + } + } + + match (kind, counter) { + (RepeatKind::OneOrMore, 0) => { + res.add_err(ExpandError::UnexpectedToken); + } + (_, 0) => { + // Collect all empty variables in subtrees + let mut vars = Vec::new(); + collect_vars(&mut vars, pattern)?; + for var in vars { + res.bindings.push_empty(&var) + } + } + _ => (), + } + Ok(()) +} + +fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult> { + let fragment = match kind { + "path" => Path, + "expr" => Expr, + "ty" => Type, + "pat" => Pattern, + "stmt" => Statement, + "block" => Block, + "meta" => MetaItem, + "item" => Item, + _ => { + let tt_result = match kind { + "ident" => input + .expect_ident() + .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) + .map_err(|()| err!("expected ident")), + "tt" => input.expect_tt().map(Some).map_err(|()| err!()), + "lifetime" => input + .expect_lifetime() + .map(|tt| Some(tt)) + .map_err(|()| err!("expected lifetime")), + "literal" => { + let neg = input.eat_char('-'); + input + .expect_literal() + .map(|literal| { + let lit = tt::Leaf::from(literal.clone()); + match neg { + None => Some(lit.into()), + Some(neg) => Some(tt::TokenTree::Subtree(tt::Subtree { + delimiter: None, + token_trees: vec![neg, lit.into()], + })), + } + }) + .map_err(|()| err!()) + } + // `vis` is optional + "vis" => match input.eat_vis() { + Some(vis) => Ok(Some(vis)), + None => Ok(None), + }, + _ => Err(ExpandError::UnexpectedToken), + }; + return tt_result.map(|it| it.map(Fragment::Tokens)).into(); + } + }; + let result = input.expect_fragment(fragment); + result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) +} + +fn collect_vars(buf: &mut Vec, pattern: &MetaTemplate) -> Result<(), ExpandError> { + for op in pattern.iter() { + match op.as_ref().map_err(|e| e.clone())? { + Op::Var { name, .. } => buf.push(name.clone()), + Op::Leaf(_) => (), + Op::Subtree(subtree) => collect_vars(buf, subtree)?, + Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?, + } + } + Ok(()) +} diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs new file mode 100644 index 000000000..82bace110 --- /dev/null +++ b/crates/mbe/src/expander/transcriber.rs @@ -0,0 +1,248 @@ +//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like +//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` + +use syntax::SmolStr; + +use super::ExpandResult; +use crate::{ + expander::{Binding, Bindings, Fragment}, + parser::{Op, RepeatKind, Separator}, + ExpandError, MetaTemplate, +}; + +impl Bindings { + fn contains(&self, name: &str) -> bool { + self.inner.contains_key(name) + } + + fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> { + let mut b = self.inner.get(name).ok_or_else(|| { + ExpandError::BindingError(format!("could not find binding `{}`", name)) + })?; + for nesting_state in nesting.iter_mut() { + nesting_state.hit = true; + b = match b { + Binding::Fragment(_) => break, + Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| { + nesting_state.at_end = true; + ExpandError::BindingError(format!("could not find nested binding `{}`", name)) + })?, + Binding::Empty => { + nesting_state.at_end = true; + return Err(ExpandError::BindingError(format!( + "could not find empty binding `{}`", + name + ))); + } + }; + } + match b { + Binding::Fragment(it) => Ok(it), + Binding::Nested(_) => Err(ExpandError::BindingError(format!( + "expected simple binding, found nested binding `{}`", + name + ))), + Binding::Empty => Err(ExpandError::BindingError(format!( + "expected simple binding, found empty binding `{}`", + name + ))), + } + } +} + +pub(super) fn transcribe( + template: &MetaTemplate, + bindings: &Bindings, +) -> ExpandResult { + assert!(template.delimiter == None); + let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; + let mut arena: Vec = Vec::new(); + expand_subtree(&mut ctx, template, &mut arena) +} + +#[derive(Debug)] +struct NestingState { + idx: usize, + /// `hit` is currently necessary to tell `expand_repeat` if it should stop + /// because there is no variable in use by the current repetition + hit: bool, + /// `at_end` is currently necessary to tell `expand_repeat` if it should stop + /// because there is no more value available for the current repetition + at_end: bool, +} + +#[derive(Debug)] +struct ExpandCtx<'a> { + bindings: &'a Bindings, + nesting: Vec, +} + +fn expand_subtree( + ctx: &mut ExpandCtx, + template: &MetaTemplate, + arena: &mut Vec, +) -> ExpandResult { + // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation + let start_elements = arena.len(); + let mut err = None; + for op in template.iter() { + let op = match op { + Ok(op) => op, + Err(e) => { + err = Some(e.clone()); + break; + } + }; + match op { + Op::Leaf(tt) => arena.push(tt.clone().into()), + Op::Subtree(tt) => { + let ExpandResult { value: tt, err: e } = expand_subtree(ctx, &tt, arena); + err = err.or(e); + arena.push(tt.into()); + } + Op::Var { name, id, .. } => { + let ExpandResult { value: fragment, err: e } = expand_var(ctx, &name, *id); + err = err.or(e); + push_fragment(arena, fragment); + } + Op::Repeat { subtree, kind, separator } => { + let ExpandResult { value: fragment, err: e } = + expand_repeat(ctx, subtree, *kind, separator, arena); + err = err.or(e); + push_fragment(arena, fragment) + } + } + } + // drain the elements added in this instance of expand_subtree + let tts = arena.drain(start_elements..arena.len()).collect(); + ExpandResult { value: tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err } +} + +fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr, id: tt::TokenId) -> ExpandResult { + // We already handle $crate case in mbe parser + debug_assert!(v != "crate"); + + if !ctx.bindings.contains(v) { + // Note that it is possible to have a `$var` inside a macro which is not bound. + // For example: + // ``` + // macro_rules! foo { + // ($a:ident, $b:ident, $c:tt) => { + // macro_rules! bar { + // ($bi:ident) => { + // fn $bi() -> u8 {$c} + // } + // } + // } + // ``` + // We just treat it a normal tokens + let tt = tt::Subtree { + delimiter: None, + token_trees: vec![ + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(), + tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(), + ], + } + .into(); + ExpandResult::ok(Fragment::Tokens(tt)) + } else { + ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( + |e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) }, + |b| ExpandResult::ok(b.clone()), + ) + } +} + +fn expand_repeat( + ctx: &mut ExpandCtx, + template: &MetaTemplate, + kind: RepeatKind, + separator: &Option, + arena: &mut Vec, +) -> ExpandResult { + let mut buf: Vec = Vec::new(); + ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a proper macro-by-example implementation + let limit = 65536; + let mut has_seps = 0; + let mut counter = 0; + + loop { + let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, arena); + let nesting_state = ctx.nesting.last_mut().unwrap(); + if nesting_state.at_end || !nesting_state.hit { + break; + } + nesting_state.idx += 1; + nesting_state.hit = false; + + counter += 1; + if counter == limit { + log::warn!("expand_tt in repeat pattern exceed limit => {:#?}\n{:#?}", template, ctx); + break; + } + + if e.is_some() { + continue; + } + + t.delimiter = None; + push_subtree(&mut buf, t); + + if let Some(ref sep) = separator { + match sep { + Separator::Ident(ident) => { + has_seps = 1; + buf.push(tt::Leaf::from(ident.clone()).into()); + } + Separator::Literal(lit) => { + has_seps = 1; + buf.push(tt::Leaf::from(lit.clone()).into()); + } + + Separator::Puncts(puncts) => { + has_seps = puncts.len(); + for punct in puncts { + buf.push(tt::Leaf::from(*punct).into()); + } + } + } + } + + if RepeatKind::ZeroOrOne == kind { + break; + } + } + + ctx.nesting.pop().unwrap(); + for _ in 0..has_seps { + buf.pop(); + } + + // Check if it is a single token subtree without any delimiter + // e.g {Delimiter:None> ['>'] /Delimiter:None>} + let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); + + if RepeatKind::OneOrMore == kind && counter == 0 { + return ExpandResult { + value: Fragment::Tokens(tt), + err: Some(ExpandError::UnexpectedToken), + }; + } + ExpandResult::ok(Fragment::Tokens(tt)) +} + +fn push_fragment(buf: &mut Vec, fragment: Fragment) { + match fragment { + Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), + Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), + } +} + +fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { + match tt.delimiter { + None => buf.extend(tt.token_trees), + _ => buf.push(tt.into()), + } +} diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 35cde5f10..bbe71ce3e 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -4,7 +4,7 @@ //! `TokenTree`s as well! mod parser; -mod mbe_expander; +mod expander; mod syntax_bridge; mod tt_iter; mod subtree_source; @@ -209,7 +209,7 @@ impl MacroRules { // apply shift let mut tt = tt.clone(); self.shift.shift_all(&mut tt); - mbe_expander::expand_rules(&self.rules, &tt) + expander::expand_rules(&self.rules, &tt) } pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { @@ -260,7 +260,7 @@ impl MacroDef { // apply shift let mut tt = tt.clone(); self.shift.shift_all(&mut tt); - mbe_expander::expand_rules(&self.rules, &tt) + expander::expand_rules(&self.rules, &tt) } pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { diff --git a/crates/mbe/src/mbe_expander.rs b/crates/mbe/src/mbe_expander.rs deleted file mode 100644 index 802c8fb0f..000000000 --- a/crates/mbe/src/mbe_expander.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! This module takes a (parsed) definition of `macro_rules` invocation, a -//! `tt::TokenTree` representing an argument of macro invocation, and produces a -//! `tt::TokenTree` for the result of the expansion. - -mod matcher; -mod transcriber; - -use rustc_hash::FxHashMap; -use syntax::SmolStr; - -use crate::{ExpandError, ExpandResult}; - -pub(crate) fn expand_rules( - rules: &[crate::Rule], - input: &tt::Subtree, -) -> ExpandResult { - let mut match_: Option<(matcher::Match, &crate::Rule)> = None; - for rule in rules { - let new_match = match matcher::match_(&rule.lhs, input) { - Ok(m) => m, - Err(_e) => { - // error in pattern parsing - continue; - } - }; - if new_match.err.is_none() { - // If we find a rule that applies without errors, we're done. - // Unconditionally returning the transcription here makes the - // `test_repeat_bad_var` test fail. - let ExpandResult { value, err: transcribe_err } = - transcriber::transcribe(&rule.rhs, &new_match.bindings); - if transcribe_err.is_none() { - return ExpandResult::ok(value); - } - } - // Use the rule if we matched more tokens, or had fewer errors - if let Some((prev_match, _)) = &match_ { - if (new_match.unmatched_tts, new_match.err_count) - < (prev_match.unmatched_tts, prev_match.err_count) - { - match_ = Some((new_match, rule)); - } - } else { - match_ = Some((new_match, rule)); - } - } - if let Some((match_, rule)) = match_ { - // if we got here, there was no match without errors - let ExpandResult { value, err: transcribe_err } = - transcriber::transcribe(&rule.rhs, &match_.bindings); - ExpandResult { value, err: match_.err.or(transcribe_err) } - } else { - ExpandResult::only_err(ExpandError::NoMatchingRule) - } -} - -/// The actual algorithm for expansion is not too hard, but is pretty tricky. -/// `Bindings` structure is the key to understanding what we are doing here. -/// -/// On the high level, it stores mapping from meta variables to the bits of -/// syntax it should be substituted with. For example, if `$e:expr` is matched -/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`. -/// -/// The tricky bit is dealing with repetitions (`$()*`). Consider this example: -/// -/// ```not_rust -/// macro_rules! foo { -/// ($($ i:ident $($ e:expr),*);*) => { -/// $(fn $ i() { $($ e);*; })* -/// } -/// } -/// foo! { foo 1,2,3; bar 4,5,6 } -/// ``` -/// -/// Here, the `$i` meta variable is matched first with `foo` and then with -/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`. -/// -/// To represent such "multi-mappings", we use a recursive structures: we map -/// variables not to values, but to *lists* of values or other lists (that is, -/// to the trees). -/// -/// For the above example, the bindings would store -/// -/// ```not_rust -/// i -> [foo, bar] -/// e -> [[1, 2, 3], [4, 5, 6]] -/// ``` -/// -/// We construct `Bindings` in the `match_lhs`. The interesting case is -/// `TokenTree::Repeat`, where we use `push_nested` to create the desired -/// nesting structure. -/// -/// The other side of the puzzle is `expand_subtree`, where we use the bindings -/// to substitute meta variables in the output template. When expanding, we -/// maintain a `nesting` stack of indices which tells us which occurrence from -/// the `Bindings` we should take. We push to the stack when we enter a -/// repetition. -/// -/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to -/// `tt::TokenTree`, where the index to select a particular `TokenTree` among -/// many is not a plain `usize`, but an `&[usize]`. -#[derive(Debug, Default)] -struct Bindings { - inner: FxHashMap, -} - -#[derive(Debug)] -enum Binding { - Fragment(Fragment), - Nested(Vec), - Empty, -} - -#[derive(Debug, Clone)] -enum Fragment { - /// token fragments are just copy-pasted into the output - Tokens(tt::TokenTree), - /// Ast fragments are inserted with fake delimiters, so as to make things - /// like `$i * 2` where `$i = 1 + 1` work as expectd. - Ast(tt::TokenTree), -} - -#[cfg(test)] -mod tests { - use syntax::{ast, AstNode}; - - use super::*; - use crate::ast_to_token_tree; - - #[test] - fn test_expand_rule() { - assert_err( - "($($i:ident);*) => ($i)", - "foo!{a}", - ExpandError::BindingError(String::from( - "expected simple binding, found nested binding `i`", - )), - ); - - // FIXME: - // Add an err test case for ($($i:ident)) => ($()) - } - - fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) { - assert_eq!( - expand_first(&create_rules(&format_macro(macro_body)), invocation).err, - Some(err) - ); - } - - fn format_macro(macro_body: &str) -> String { - format!( - " - macro_rules! foo {{ - {} - }} -", - macro_body - ) - } - - fn create_rules(macro_definition: &str) -> crate::MacroRules { - let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap(); - let macro_definition = - source_file.syntax().descendants().find_map(ast::MacroRules::cast).unwrap(); - - let (definition_tt, _) = - ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); - crate::MacroRules::parse(&definition_tt).unwrap() - } - - fn expand_first(rules: &crate::MacroRules, invocation: &str) -> ExpandResult { - let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); - let macro_invocation = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (invocation_tt, _) = - ast_to_token_tree(¯o_invocation.token_tree().unwrap()).unwrap(); - - expand_rules(&rules.rules, &invocation_tt) - } -} diff --git a/crates/mbe/src/mbe_expander/matcher.rs b/crates/mbe/src/mbe_expander/matcher.rs deleted file mode 100644 index d32e60521..000000000 --- a/crates/mbe/src/mbe_expander/matcher.rs +++ /dev/null @@ -1,502 +0,0 @@ -//! FIXME: write short doc here - -use crate::{ - mbe_expander::{Binding, Bindings, Fragment}, - parser::{Op, RepeatKind, Separator}, - subtree_source::SubtreeTokenSource, - tt_iter::TtIter, - ExpandError, MetaTemplate, -}; - -use super::ExpandResult; -use parser::{FragmentKind::*, TreeSink}; -use syntax::{SmolStr, SyntaxKind}; -use tt::buffer::{Cursor, TokenBuffer}; - -impl Bindings { - fn push_optional(&mut self, name: &SmolStr) { - // FIXME: Do we have a better way to represent an empty token ? - // Insert an empty subtree for empty token - let tt = tt::Subtree::default().into(); - self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); - } - - fn push_empty(&mut self, name: &SmolStr) { - self.inner.insert(name.clone(), Binding::Empty); - } - - fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> { - for (key, value) in nested.inner { - if !self.inner.contains_key(&key) { - self.inner.insert(key.clone(), Binding::Nested(Vec::new())); - } - match self.inner.get_mut(&key) { - Some(Binding::Nested(it)) => { - // insert empty nested bindings before this one - while it.len() < idx { - it.push(Binding::Nested(vec![])); - } - it.push(value); - } - _ => { - return Err(ExpandError::BindingError(format!( - "could not find binding `{}`", - key - ))); - } - } - } - Ok(()) - } -} - -macro_rules! err { - () => { - ExpandError::BindingError(format!("")) - }; - ($($tt:tt)*) => { - ExpandError::BindingError(format!($($tt)*)) - }; -} - -#[derive(Debug, Default)] -pub(super) struct Match { - pub(super) bindings: Bindings, - /// We currently just keep the first error and count the rest to compare matches. - pub(super) err: Option, - pub(super) err_count: usize, - /// How many top-level token trees were left to match. - pub(super) unmatched_tts: usize, -} - -impl Match { - pub(super) fn add_err(&mut self, err: ExpandError) { - let prev_err = self.err.take(); - self.err = prev_err.or(Some(err)); - self.err_count += 1; - } -} - -// General note: These functions have two channels to return errors, a `Result` -// return value and the `&mut Match`. The returned Result is for pattern parsing -// errors; if a branch of the macro definition doesn't parse, it doesn't make -// sense to try using it. Matching errors are added to the `Match`. It might -// make sense to make pattern parsing a separate step? - -pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Result { - assert!(pattern.delimiter == None); - - let mut res = Match::default(); - let mut src = TtIter::new(src); - - match_subtree(&mut res, pattern, &mut src)?; - - if src.len() > 0 { - res.unmatched_tts += src.len(); - res.add_err(err!("leftover tokens")); - } - - Ok(res) -} - -fn match_subtree( - res: &mut Match, - pattern: &MetaTemplate, - src: &mut TtIter, -) -> Result<(), ExpandError> { - for op in pattern.iter() { - match op.as_ref().map_err(|err| err.clone())? { - Op::Leaf(lhs) => { - let rhs = match src.expect_leaf() { - Ok(l) => l, - Err(()) => { - res.add_err(err!("expected leaf: `{}`", lhs)); - continue; - } - }; - match (lhs, rhs) { - ( - tt::Leaf::Punct(tt::Punct { char: lhs, .. }), - tt::Leaf::Punct(tt::Punct { char: rhs, .. }), - ) if lhs == rhs => (), - ( - tt::Leaf::Ident(tt::Ident { text: lhs, .. }), - tt::Leaf::Ident(tt::Ident { text: rhs, .. }), - ) if lhs == rhs => (), - ( - tt::Leaf::Literal(tt::Literal { text: lhs, .. }), - tt::Leaf::Literal(tt::Literal { text: rhs, .. }), - ) if lhs == rhs => (), - _ => { - res.add_err(ExpandError::UnexpectedToken); - } - } - } - Op::Subtree(lhs) => { - let rhs = match src.expect_subtree() { - Ok(s) => s, - Err(()) => { - res.add_err(err!("expected subtree")); - continue; - } - }; - if lhs.delimiter_kind() != rhs.delimiter_kind() { - res.add_err(err!("mismatched delimiter")); - continue; - } - let mut src = TtIter::new(rhs); - match_subtree(res, lhs, &mut src)?; - if src.len() > 0 { - res.add_err(err!("leftover tokens")); - } - } - Op::Var { name, kind, .. } => { - let kind = match kind { - Some(k) => k, - None => { - res.add_err(ExpandError::UnexpectedToken); - continue; - } - }; - let ExpandResult { value: matched, err: match_err } = - match_meta_var(kind.as_str(), src); - match matched { - Some(fragment) => { - res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); - } - None if match_err.is_none() => res.bindings.push_optional(name), - _ => {} - } - if let Some(err) = match_err { - res.add_err(err); - } - } - Op::Repeat { subtree, kind, separator } => { - match_repeat(res, subtree, *kind, separator, src)?; - } - } - } - Ok(()) -} - -impl<'a> TtIter<'a> { - fn eat_separator(&mut self, separator: &Separator) -> bool { - let mut fork = self.clone(); - let ok = match separator { - Separator::Ident(lhs) => match fork.expect_ident() { - Ok(rhs) => rhs.text == lhs.text, - _ => false, - }, - Separator::Literal(lhs) => match fork.expect_literal() { - Ok(rhs) => match rhs { - tt::Leaf::Literal(rhs) => rhs.text == lhs.text, - tt::Leaf::Ident(rhs) => rhs.text == lhs.text, - tt::Leaf::Punct(_) => false, - }, - _ => false, - }, - Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { - Ok(rhs) => rhs.char == lhs.char, - _ => false, - }), - }; - if ok { - *self = fork; - } - ok - } - - pub(crate) fn expect_tt(&mut self) -> Result { - match self.peek_n(0) { - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { - return self.expect_lifetime(); - } - _ => (), - } - - let tt = self.next().ok_or_else(|| ())?.clone(); - let punct = match tt { - tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { - punct - } - _ => return Ok(tt), - }; - - let (second, third) = match (self.peek_n(0), self.peek_n(1)) { - ( - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))), - ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)), - (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None), - _ => return Ok(tt), - }; - - match (punct.char, second, third) { - ('.', '.', Some('.')) - | ('.', '.', Some('=')) - | ('<', '<', Some('=')) - | ('>', '>', Some('=')) => { - let tt2 = self.next().unwrap().clone(); - let tt3 = self.next().unwrap().clone(); - Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into()) - } - ('-', '=', _) - | ('-', '>', _) - | (':', ':', _) - | ('!', '=', _) - | ('.', '.', _) - | ('*', '=', _) - | ('/', '=', _) - | ('&', '&', _) - | ('&', '=', _) - | ('%', '=', _) - | ('^', '=', _) - | ('+', '=', _) - | ('<', '<', _) - | ('<', '=', _) - | ('=', '=', _) - | ('=', '>', _) - | ('>', '=', _) - | ('>', '>', _) - | ('|', '=', _) - | ('|', '|', _) => { - let tt2 = self.next().unwrap().clone(); - Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into()) - } - _ => Ok(tt), - } - } - - pub(crate) fn expect_lifetime(&mut self) -> Result { - let punct = self.expect_punct()?; - if punct.char != '\'' { - return Err(()); - } - let ident = self.expect_ident()?; - - Ok(tt::Subtree { - delimiter: None, - token_trees: vec![ - tt::Leaf::Punct(*punct).into(), - tt::Leaf::Ident(ident.clone()).into(), - ], - } - .into()) - } - - pub(crate) fn expect_fragment( - &mut self, - fragment_kind: parser::FragmentKind, - ) -> ExpandResult> { - pub(crate) struct OffsetTokenSink<'a> { - pub(crate) cursor: Cursor<'a>, - pub(crate) error: bool, - } - - impl<'a> TreeSink for OffsetTokenSink<'a> { - fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { - if kind == SyntaxKind::LIFETIME_IDENT { - n_tokens = 2; - } - for _ in 0..n_tokens { - self.cursor = self.cursor.bump_subtree(); - } - } - fn start_node(&mut self, _kind: SyntaxKind) {} - fn finish_node(&mut self) {} - fn error(&mut self, _error: parser::ParseError) { - self.error = true; - } - } - - let buffer = TokenBuffer::from_tokens(&self.inner.as_slice()); - let mut src = SubtreeTokenSource::new(&buffer); - let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; - - parser::parse_fragment(&mut src, &mut sink, fragment_kind); - - let mut err = None; - if !sink.cursor.is_root() || sink.error { - err = Some(err!("expected {:?}", fragment_kind)); - } - - let mut curr = buffer.begin(); - let mut res = vec![]; - - if sink.cursor.is_root() { - while curr != sink.cursor { - if let Some(token) = curr.token_tree() { - res.push(token); - } - curr = curr.bump(); - } - } - self.inner = self.inner.as_slice()[res.len()..].iter(); - if res.len() == 0 && err.is_none() { - err = Some(err!("no tokens consumed")); - } - let res = match res.len() { - 1 => Some(res[0].cloned()), - 0 => None, - _ => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, - token_trees: res.into_iter().map(|it| it.cloned()).collect(), - })), - }; - ExpandResult { value: res, err } - } - - pub(crate) fn eat_vis(&mut self) -> Option { - let mut fork = self.clone(); - match fork.expect_fragment(Visibility) { - ExpandResult { value: tt, err: None } => { - *self = fork; - tt - } - ExpandResult { value: _, err: Some(_) } => None, - } - } - - pub(crate) fn eat_char(&mut self, c: char) -> Option { - let mut fork = self.clone(); - match fork.expect_char(c) { - Ok(_) => { - let tt = self.next().cloned(); - *self = fork; - tt - } - Err(_) => None, - } - } -} - -pub(super) fn match_repeat( - res: &mut Match, - pattern: &MetaTemplate, - kind: RepeatKind, - separator: &Option, - src: &mut TtIter, -) -> Result<(), ExpandError> { - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a proper macro-by-example implementation - let mut limit = 65536; - let mut counter = 0; - - for i in 0.. { - let mut fork = src.clone(); - - if let Some(separator) = &separator { - if i != 0 && !fork.eat_separator(separator) { - break; - } - } - - let mut nested = Match::default(); - match_subtree(&mut nested, pattern, &mut fork)?; - if nested.err.is_none() { - limit -= 1; - if limit == 0 { - log::warn!( - "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", - pattern, - src, - kind, - separator - ); - break; - } - *src = fork; - - if let Err(err) = res.bindings.push_nested(counter, nested.bindings) { - res.add_err(err); - } - counter += 1; - if counter == 1 { - if let RepeatKind::ZeroOrOne = kind { - break; - } - } - } else { - break; - } - } - - match (kind, counter) { - (RepeatKind::OneOrMore, 0) => { - res.add_err(ExpandError::UnexpectedToken); - } - (_, 0) => { - // Collect all empty variables in subtrees - let mut vars = Vec::new(); - collect_vars(&mut vars, pattern)?; - for var in vars { - res.bindings.push_empty(&var) - } - } - _ => (), - } - Ok(()) -} - -fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult> { - let fragment = match kind { - "path" => Path, - "expr" => Expr, - "ty" => Type, - "pat" => Pattern, - "stmt" => Statement, - "block" => Block, - "meta" => MetaItem, - "item" => Item, - _ => { - let tt_result = match kind { - "ident" => input - .expect_ident() - .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) - .map_err(|()| err!("expected ident")), - "tt" => input.expect_tt().map(Some).map_err(|()| err!()), - "lifetime" => input - .expect_lifetime() - .map(|tt| Some(tt)) - .map_err(|()| err!("expected lifetime")), - "literal" => { - let neg = input.eat_char('-'); - input - .expect_literal() - .map(|literal| { - let lit = tt::Leaf::from(literal.clone()); - match neg { - None => Some(lit.into()), - Some(neg) => Some(tt::TokenTree::Subtree(tt::Subtree { - delimiter: None, - token_trees: vec![neg, lit.into()], - })), - } - }) - .map_err(|()| err!()) - } - // `vis` is optional - "vis" => match input.eat_vis() { - Some(vis) => Ok(Some(vis)), - None => Ok(None), - }, - _ => Err(ExpandError::UnexpectedToken), - }; - return tt_result.map(|it| it.map(Fragment::Tokens)).into(); - } - }; - let result = input.expect_fragment(fragment); - result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) -} - -fn collect_vars(buf: &mut Vec, pattern: &MetaTemplate) -> Result<(), ExpandError> { - for op in pattern.iter() { - match op.as_ref().map_err(|e| e.clone())? { - Op::Var { name, .. } => buf.push(name.clone()), - Op::Leaf(_) => (), - Op::Subtree(subtree) => collect_vars(buf, subtree)?, - Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?, - } - } - Ok(()) -} diff --git a/crates/mbe/src/mbe_expander/transcriber.rs b/crates/mbe/src/mbe_expander/transcriber.rs deleted file mode 100644 index 59a3c80a8..000000000 --- a/crates/mbe/src/mbe_expander/transcriber.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like -//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` - -use syntax::SmolStr; - -use super::ExpandResult; -use crate::{ - mbe_expander::{Binding, Bindings, Fragment}, - parser::{Op, RepeatKind, Separator}, - ExpandError, MetaTemplate, -}; - -impl Bindings { - fn contains(&self, name: &str) -> bool { - self.inner.contains_key(name) - } - - fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> { - let mut b = self.inner.get(name).ok_or_else(|| { - ExpandError::BindingError(format!("could not find binding `{}`", name)) - })?; - for nesting_state in nesting.iter_mut() { - nesting_state.hit = true; - b = match b { - Binding::Fragment(_) => break, - Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| { - nesting_state.at_end = true; - ExpandError::BindingError(format!("could not find nested binding `{}`", name)) - })?, - Binding::Empty => { - nesting_state.at_end = true; - return Err(ExpandError::BindingError(format!( - "could not find empty binding `{}`", - name - ))); - } - }; - } - match b { - Binding::Fragment(it) => Ok(it), - Binding::Nested(_) => Err(ExpandError::BindingError(format!( - "expected simple binding, found nested binding `{}`", - name - ))), - Binding::Empty => Err(ExpandError::BindingError(format!( - "expected simple binding, found empty binding `{}`", - name - ))), - } - } -} - -pub(super) fn transcribe( - template: &MetaTemplate, - bindings: &Bindings, -) -> ExpandResult { - assert!(template.delimiter == None); - let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; - let mut arena: Vec = Vec::new(); - expand_subtree(&mut ctx, template, &mut arena) -} - -#[derive(Debug)] -struct NestingState { - idx: usize, - /// `hit` is currently necessary to tell `expand_repeat` if it should stop - /// because there is no variable in use by the current repetition - hit: bool, - /// `at_end` is currently necessary to tell `expand_repeat` if it should stop - /// because there is no more value available for the current repetition - at_end: bool, -} - -#[derive(Debug)] -struct ExpandCtx<'a> { - bindings: &'a Bindings, - nesting: Vec, -} - -fn expand_subtree( - ctx: &mut ExpandCtx, - template: &MetaTemplate, - arena: &mut Vec, -) -> ExpandResult { - // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation - let start_elements = arena.len(); - let mut err = None; - for op in template.iter() { - let op = match op { - Ok(op) => op, - Err(e) => { - err = Some(e.clone()); - break; - } - }; - match op { - Op::Leaf(tt) => arena.push(tt.clone().into()), - Op::Subtree(tt) => { - let ExpandResult { value: tt, err: e } = expand_subtree(ctx, &tt, arena); - err = err.or(e); - arena.push(tt.into()); - } - Op::Var { name, id, .. } => { - let ExpandResult { value: fragment, err: e } = expand_var(ctx, &name, *id); - err = err.or(e); - push_fragment(arena, fragment); - } - Op::Repeat { subtree, kind, separator } => { - let ExpandResult { value: fragment, err: e } = - expand_repeat(ctx, subtree, *kind, separator, arena); - err = err.or(e); - push_fragment(arena, fragment) - } - } - } - // drain the elements added in this instance of expand_subtree - let tts = arena.drain(start_elements..arena.len()).collect(); - ExpandResult { value: tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err } -} - -fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr, id: tt::TokenId) -> ExpandResult { - // We already handle $crate case in mbe parser - debug_assert!(v != "crate"); - - if !ctx.bindings.contains(v) { - // Note that it is possible to have a `$var` inside a macro which is not bound. - // For example: - // ``` - // macro_rules! foo { - // ($a:ident, $b:ident, $c:tt) => { - // macro_rules! bar { - // ($bi:ident) => { - // fn $bi() -> u8 {$c} - // } - // } - // } - // ``` - // We just treat it a normal tokens - let tt = tt::Subtree { - delimiter: None, - token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, id }).into(), - tt::Leaf::from(tt::Ident { text: v.clone(), id }).into(), - ], - } - .into(); - ExpandResult::ok(Fragment::Tokens(tt)) - } else { - ctx.bindings.get(&v, &mut ctx.nesting).map_or_else( - |e| ExpandResult { value: Fragment::Tokens(tt::TokenTree::empty()), err: Some(e) }, - |b| ExpandResult::ok(b.clone()), - ) - } -} - -fn expand_repeat( - ctx: &mut ExpandCtx, - template: &MetaTemplate, - kind: RepeatKind, - separator: &Option, - arena: &mut Vec, -) -> ExpandResult { - let mut buf: Vec = Vec::new(); - ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a proper macro-by-example implementation - let limit = 65536; - let mut has_seps = 0; - let mut counter = 0; - - loop { - let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, arena); - let nesting_state = ctx.nesting.last_mut().unwrap(); - if nesting_state.at_end || !nesting_state.hit { - break; - } - nesting_state.idx += 1; - nesting_state.hit = false; - - counter += 1; - if counter == limit { - log::warn!("expand_tt in repeat pattern exceed limit => {:#?}\n{:#?}", template, ctx); - break; - } - - if e.is_some() { - continue; - } - - t.delimiter = None; - push_subtree(&mut buf, t); - - if let Some(ref sep) = separator { - match sep { - Separator::Ident(ident) => { - has_seps = 1; - buf.push(tt::Leaf::from(ident.clone()).into()); - } - Separator::Literal(lit) => { - has_seps = 1; - buf.push(tt::Leaf::from(lit.clone()).into()); - } - - Separator::Puncts(puncts) => { - has_seps = puncts.len(); - for punct in puncts { - buf.push(tt::Leaf::from(*punct).into()); - } - } - } - } - - if RepeatKind::ZeroOrOne == kind { - break; - } - } - - ctx.nesting.pop().unwrap(); - for _ in 0..has_seps { - buf.pop(); - } - - // Check if it is a single token subtree without any delimiter - // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); - - if RepeatKind::OneOrMore == kind && counter == 0 { - return ExpandResult { - value: Fragment::Tokens(tt), - err: Some(ExpandError::UnexpectedToken), - }; - } - ExpandResult::ok(Fragment::Tokens(tt)) -} - -fn push_fragment(buf: &mut Vec, fragment: Fragment) { - match fragment { - Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), - } -} - -fn push_subtree(buf: &mut Vec, tt: tt::Subtree) { - match tt.delimiter { - None => buf.extend(tt.token_trees), - _ => buf.push(tt.into()), - } -} -- cgit v1.2.3