11 files changed, 1530 insertions, 715 deletions

diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml
index af80e2be3..bb2656a80 100644
--- a/crates/mbe/Cargo.toml
+++ b/crates/mbe/Cargo.toml
@@ -17,6 +17,8 @@ log = "0.4.8"
 syntax = { path = "../syntax", version = "0.0.0" }
 parser = { path = "../parser", version = "0.0.0" }
 tt = { path = "../tt", version = "0.0.0" }
+test_utils = { path = "../test_utils", version = "0.0.0" }
+stdx = { path = "../stdx", version = "0.0.0" }
 
 [dev-dependencies]
-test_utils = { path = "../test_utils" }
+profile = { path = "../profile" }
diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs
new file mode 100644
index 000000000..503ad1355
--- /dev/null
+++ b/crates/mbe/src/benchmark.rs
@@ -0,0 +1,225 @@
+//! This module add real world mbe example for benchmark tests
+
+use rustc_hash::FxHashMap;
+use syntax::{
+    ast::{self, NameOwner},
+    AstNode, SmolStr,
+};
+use test_utils::{bench, bench_fixture, skip_slow_tests};
+
+use crate::{
+    ast_to_token_tree,
+    parser::{Op, RepeatKind, Separator},
+    MacroRules,
+};
+
+#[test]
+fn benchmark_parse_macro_rules() {
+    if skip_slow_tests() {
+        return;
+    }
+    let rules = macro_rules_fixtures_tt();
+    let hash: usize = {
+        let _pt = bench("mbe parse macro rules");
+        rules.values().map(|it| MacroRules::parse(it).unwrap().rules.len()).sum()
+    };
+    assert_eq!(hash, 1144);
+}
+
+#[test]
+fn benchmark_expand_macro_rules() {
+    if skip_slow_tests() {
+        return;
+    }
+    let rules = macro_rules_fixtures();
+    let invocations = invocation_fixtures(&rules);
+
+    let hash: usize = {
+        let _pt = bench("mbe expand macro rules");
+        invocations
+            .into_iter()
+            .map(|(id, tt)| {
+                let res = rules[&id].expand(&tt);
+                assert!(res.err.is_none());
+                res.value.token_trees.len()
+            })
+            .sum()
+    };
+    assert_eq!(hash, 69413);
+}
+
+fn macro_rules_fixtures() -> FxHashMap<String, MacroRules> {
+    macro_rules_fixtures_tt()
+        .into_iter()
+        .map(|(id, tt)| (id, MacroRules::parse(&tt).unwrap()))
+        .collect()
+}
+
+fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree> {
+    let fixture = bench_fixture::numerous_macro_rules();
+    let source_file = ast::SourceFile::parse(&fixture).ok().unwrap();
+
+    source_file
+        .syntax()
+        .descendants()
+        .filter_map(ast::MacroRules::cast)
+        .map(|rule| {
+            let id = rule.name().unwrap().to_string();
+            let (def_tt, _) = ast_to_token_tree(&rule.token_tree().unwrap()).unwrap();
+            (id, def_tt)
+        })
+        .collect()
+}
+
+/// Generate random invocation fixtures from rules
+fn invocation_fixtures(rules: &FxHashMap<String, MacroRules>) -> Vec<(String, tt::Subtree)> {
+    let mut seed = 123456789;
+    let mut res = Vec::new();
+
+    for (name, it) in rules {
+        for rule in &it.rules {
+            // Generate twice
+            for _ in 0..2 {
+                // The input are generated by filling the `Op` randomly.
+                // However, there are some cases generated are ambiguous for expanding, for example:
+                // ```rust
+                // macro_rules! m {
+                //    ($($t:ident),* as $ty:ident) => {}
+                // }
+                // m!(as u32);  // error: local ambiguity: multiple parsing options: built-in NTs ident ('t') or 1 other option.
+                // ```
+                //
+                // So we just skip any error cases and try again
+                let mut try_cnt = 0;
+                loop {
+                    let mut subtree = tt::Subtree::default();
+                    for op in rule.lhs.iter() {
+                        collect_from_op(op, &mut subtree, &mut seed);
+                    }
+                    if it.expand(&subtree).err.is_none() {
+                        res.push((name.clone(), subtree));
+                        break;
+                    }
+                    try_cnt += 1;
+                    if try_cnt > 100 {
+                        panic!("invocaton fixture {} cannot be generated.\n", name);
+                    }
+                }
+            }
+        }
+    }
+    return res;
+
+    fn collect_from_op(op: &Op, parent: &mut tt::Subtree, seed: &mut usize) {
+        return match op {
+            Op::Var { kind, .. } => match kind.as_ref().map(|it| it.as_str()) {
+                Some("ident") => parent.token_trees.push(make_ident("foo")),
+                Some("ty") => parent.token_trees.push(make_ident("Foo")),
+                Some("tt") => parent.token_trees.push(make_ident("foo")),
+                Some("vis") => parent.token_trees.push(make_ident("pub")),
+                Some("pat") => parent.token_trees.push(make_ident("foo")),
+                Some("path") => parent.token_trees.push(make_ident("foo")),
+                Some("literal") => parent.token_trees.push(make_literal("1")),
+                Some("expr") => parent.token_trees.push(make_ident("foo").into()),
+                Some("lifetime") => {
+                    parent.token_trees.push(make_punct('\''));
+                    parent.token_trees.push(make_ident("a"));
+                }
+                Some("block") => {
+                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None))
+                }
+                Some("item") => {
+                    parent.token_trees.push(make_ident("fn"));
+                    parent.token_trees.push(make_ident("foo"));
+                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
+                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Brace, None));
+                }
+                Some("meta") => {
+                    parent.token_trees.push(make_ident("foo"));
+                    parent.token_trees.push(make_subtree(tt::DelimiterKind::Parenthesis, None));
+                }
+
+                None => (),
+                Some(kind) => panic!("Unhandled kind {}", kind),
+            },
+            Op::Leaf(leaf) => parent.token_trees.push(leaf.clone().into()),
+            Op::Repeat { tokens, kind, separator } => {
+                let max = 10;
+                let cnt = match kind {
+                    RepeatKind::ZeroOrMore => rand(seed) % max,
+                    RepeatKind::OneOrMore => 1 + rand(seed) % max,
+                    RepeatKind::ZeroOrOne => rand(seed) % 2,
+                };
+                for i in 0..cnt {
+                    for it in tokens.iter() {
+                        collect_from_op(it, parent, seed);
+                    }
+                    if i + 1 != cnt {
+                        if let Some(sep) = separator {
+                            match sep {
+                                Separator::Literal(it) => parent
+                                    .token_trees
+                                    .push(tt::Leaf::Literal(it.clone().into()).into()),
+                                Separator::Ident(it) => parent
+                                    .token_trees
+                                    .push(tt::Leaf::Ident(it.clone().into()).into()),
+                                Separator::Puncts(puncts) => {
+                                    for it in puncts {
+                                        parent
+                                            .token_trees
+                                            .push(tt::Leaf::Punct(it.clone().into()).into())
+                                    }
+                                }
+                            };
+                        }
+                    }
+                }
+            }
+            Op::Subtree { tokens, delimiter } => {
+                let mut subtree =
+                    tt::Subtree { delimiter: delimiter.clone(), token_trees: Vec::new() };
+                tokens.iter().for_each(|it| {
+                    collect_from_op(it, &mut subtree, seed);
+                });
+                parent.token_trees.push(subtree.into());
+            }
+        };
+
+        // Simple linear congruential generator for determistic result
+        fn rand(seed: &mut usize) -> usize {
+            let a = 1664525;
+            let c = 1013904223;
+            *seed = usize::wrapping_add(usize::wrapping_mul(*seed, a), c);
+            return *seed;
+        }
+        fn make_ident(ident: &str) -> tt::TokenTree {
+            tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), text: SmolStr::new(ident) })
+                .into()
+        }
+        fn make_punct(char: char) -> tt::TokenTree {
+            tt::Leaf::Punct(tt::Punct {
+                id: tt::TokenId::unspecified(),
+                char,
+                spacing: tt::Spacing::Alone,
+            })
+            .into()
+        }
+        fn make_literal(lit: &str) -> tt::TokenTree {
+            tt::Leaf::Literal(tt::Literal {
+                id: tt::TokenId::unspecified(),
+                text: SmolStr::new(lit),
+            })
+            .into()
+        }
+        fn make_subtree(
+            kind: tt::DelimiterKind,
+            token_trees: Option<Vec<tt::TokenTree>>,
+        ) -> tt::TokenTree {
+            tt::Subtree {
+                delimiter: Some(tt::Delimiter { id: tt::TokenId::unspecified(), kind }),
+                token_trees: token_trees.unwrap_or_default(),
+            }
+            .into()
+        }
+    }
+}
diff --git a/crates/mbe/src/mbe_expander.rs b/crates/mbe/src/expander.rs
index a80b73db4..2efff8f52 100644
--- a/crates/mbe/src/mbe_expander.rs
+++ b/crates/mbe/src/expander.rs
@@ -5,25 +5,19 @@
 mod matcher;
 mod transcriber;
 
-use rustc_hash::FxHashMap;
+use smallvec::SmallVec;
 use syntax::SmolStr;
 
 use crate::{ExpandError, ExpandResult};
 
-pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
-    expand_rules(&rules.rules, input)
-}
-
-fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+pub(crate) fn expand_rules(
+    rules: &[crate::Rule],
+    input: &tt::Subtree,
+) -> ExpandResult<tt::Subtree> {
     let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
     for rule in rules {
-        let new_match = match matcher::match_(&rule.lhs, input) {
-            Ok(m) => m,
-            Err(_e) => {
-                // error in pattern parsing
-                continue;
-            }
-        };
+        let new_match = matcher::match_(&rule.lhs, input);
+
         if new_match.err.is_none() {
             // If we find a rule that applies without errors, we're done.
             // Unconditionally returning the transcription here makes the
@@ -34,10 +28,10 @@ fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult<tt::
                 return ExpandResult::ok(value);
             }
         }
-        // Use the rule if we matched more tokens, or had fewer errors
+        // Use the rule if we matched more tokens, or bound variables count
         if let Some((prev_match, _)) = &match_ {
-            if (new_match.unmatched_tts, new_match.err_count)
-                < (prev_match.unmatched_tts, prev_match.err_count)
+            if (new_match.unmatched_tts, -(new_match.bound_count as i32))
+                < (prev_match.unmatched_tts, -(prev_match.bound_count as i32))
             {
                 match_ = Some((new_match, rule));
             }
@@ -100,19 +94,19 @@ fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult<tt::
 /// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
 /// `tt::TokenTree`, where the index to select a particular `TokenTree` among
 /// many is not a plain `usize`, but an `&[usize]`.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
 struct Bindings {
-    inner: FxHashMap<SmolStr, Binding>,
+    inner: SmallVec<[(SmolStr, Binding); 4]>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 enum Binding {
     Fragment(Fragment),
     Nested(Vec<Binding>),
     Empty,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 enum Fragment {
     /// token fragments are just copy-pasted into the output
     Tokens(tt::TokenTree),
diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs
new file mode 100644
index 000000000..9d3d28055
--- /dev/null
+++ b/crates/mbe/src/expander/matcher.rs
@@ -0,0 +1,737 @@
+//! An NFA-based parser, which is porting from rustc mbe parsing code
+//!
+//! See https://github.com/rust-lang/rust/blob/70b18bc2cbac4712020019f5bf57c00905373205/compiler/rustc_expand/src/mbe/macro_parser.rs
+//! Here is a quick intro to how the parser works, copied from rustc:
+//!
+//! A 'position' is a dot in the middle of a matcher, usually represented as a
+//! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
+//!
+//! The parser walks through the input a character at a time, maintaining a list
+//! of threads consistent with the current position in the input string: `cur_items`.
+//!
+//! As it processes them, it fills up `eof_items` with threads that would be valid if
+//! the macro invocation is now over, `bb_items` with threads that are waiting on
+//! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting
+//! on a particular token. Most of the logic concerns moving the · through the
+//! repetitions indicated by Kleene stars. The rules for moving the · without
+//! consuming any input are called epsilon transitions. It only advances or calls
+//! out to the real Rust parser when no `cur_items` threads remain.
+//!
+//! Example:
+//!
+//! ```text, ignore
+//! Start parsing a a a a b against [· a $( a )* a b].
+//!
+//! Remaining input: a a a a b
+//! next: [· a $( a )* a b]
+//!
+//! - - - Advance over an a. - - -
+//!
+//! Remaining input: a a a b
+//! cur: [a · $( a )* a b]
+//! Descend/Skip (first item).
+//! next: [a $( · a )* a b]  [a $( a )* · a b].
+//!
+//! - - - Advance over an a. - - -
+//!
+//! Remaining input: a a b
+//! cur: [a $( a · )* a b]  [a $( a )* a · b]
+//! Follow epsilon transition: Finish/Repeat (first item)
+//! next: [a $( a )* · a b]  [a $( · a )* a b]  [a $( a )* a · b]
+//!
+//! - - - Advance over an a. - - - (this looks exactly like the last step)
+//!
+//! Remaining input: a b
+//! cur: [a $( a · )* a b]  [a $( a )* a · b]
+//! Follow epsilon transition: Finish/Repeat (first item)
+//! next: [a $( a )* · a b]  [a $( · a )* a b]  [a $( a )* a · b]
+//!
+//! - - - Advance over an a. - - - (this looks exactly like the last step)
+//!
+//! Remaining input: b
+//! cur: [a $( a · )* a b]  [a $( a )* a · b]
+//! Follow epsilon transition: Finish/Repeat (first item)
+//! next: [a $( a )* · a b]  [a $( · a )* a b]  [a $( a )* a · b]
+//!
+//! - - - Advance over a b. - - -
+//!
+//! Remaining input: ''
+//! eof: [a $( a )* a b ·]
+//! ```
+
+use crate::{
+    expander::{Binding, Bindings, Fragment},
+    parser::{Op, OpDelimited, OpDelimitedIter, RepeatKind, Separator},
+    tt_iter::TtIter,
+    ExpandError, MetaTemplate,
+};
+
+use super::ExpandResult;
+use parser::FragmentKind::*;
+use smallvec::{smallvec, SmallVec};
+use syntax::SmolStr;
+
+impl Bindings {
+    fn push_optional(&mut self, name: &SmolStr) {
+        // FIXME: Do we have a better way to represent an empty token ?
+        // Insert an empty subtree for empty token
+        let tt = tt::Subtree::default().into();
+        self.inner.push((name.clone(), Binding::Fragment(Fragment::Tokens(tt))));
+    }
+
+    fn push_empty(&mut self, name: &SmolStr) {
+        self.inner.push((name.clone(), Binding::Empty));
+    }
+
+    fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
+        for (key, value) in nested.inner {
+            if self.get_mut(&key).is_none() {
+                self.inner.push((key.clone(), Binding::Nested(Vec::new())));
+            }
+            match self.get_mut(&key) {
+                Some(Binding::Nested(it)) => {
+                    // insert empty nested bindings before this one
+                    while it.len() < idx {
+                        it.push(Binding::Nested(vec![]));
+                    }
+                    it.push(value);
+                }
+                _ => {
+                    return Err(ExpandError::BindingError(format!(
+                        "could not find binding `{}`",
+                        key
+                    )));
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn get_mut(&mut self, name: &str) -> Option<&mut Binding> {
+        self.inner.iter_mut().find_map(|(n, b)| if n == name { Some(b) } else { None })
+    }
+
+    fn bindings(&self) -> impl Iterator<Item = &Binding> {
+        self.inner.iter().map(|(_, b)| b)
+    }
+}
+
+macro_rules! err {
+    () => {
+        ExpandError::BindingError(format!(""))
+    };
+    ($($tt:tt)*) => {
+        ExpandError::BindingError(format!($($tt)*))
+    };
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub(super) struct Match {
+    pub(super) bindings: Bindings,
+    /// We currently just keep the first error and count the rest to compare matches.
+    pub(super) err: Option<ExpandError>,
+    pub(super) err_count: usize,
+    /// How many top-level token trees were left to match.
+    pub(super) unmatched_tts: usize,
+    /// The number of bound variables
+    pub(super) bound_count: usize,
+}
+
+impl Match {
+    fn add_err(&mut self, err: ExpandError) {
+        let prev_err = self.err.take();
+        self.err = prev_err.or(Some(err));
+        self.err_count += 1;
+    }
+}
+
+/// Matching errors are added to the `Match`.
+pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree) -> Match {
+    let mut res = match_loop(pattern, &input);
+    res.bound_count = count(res.bindings.bindings());
+    return res;
+
+    fn count<'a>(bindings: impl Iterator<Item = &'a Binding>) -> usize {
+        bindings
+            .map(|it| match it {
+                Binding::Fragment(_) => 1,
+                Binding::Empty => 1,
+                Binding::Nested(it) => count(it.iter()),
+            })
+            .sum()
+    }
+}
+
+#[derive(Debug, Clone)]
+struct MatchState<'t> {
+    /// The position of the "dot" in this matcher
+    dot: OpDelimitedIter<'t>,
+
+    /// Token subtree stack
+    /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
+    /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
+    /// that where the bottom of the stack is the outermost matcher.
+    stack: SmallVec<[OpDelimitedIter<'t>; 4]>,
+
+    /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
+    /// before we enter the repetition.
+    up: Option<Box<MatchState<'t>>>,
+
+    /// The separator if we are in a repetition.
+    sep: Option<Separator>,
+
+    /// The KleeneOp of this sequence if we are in a repetition.
+    sep_kind: Option<RepeatKind>,
+
+    /// Number of tokens of seperator parsed
+    sep_parsed: Option<usize>,
+
+    /// Matched meta variables bindings
+    bindings: SmallVec<[Bindings; 4]>,
+
+    /// Cached result of meta variable parsing
+    meta_result: Option<(TtIter<'t>, ExpandResult<Option<Fragment>>)>,
+
+    /// Is error occuried in this state, will `poised` to "parent"
+    is_error: bool,
+}
+
+/// Process the matcher positions of `cur_items` until it is empty. In the process, this will
+/// produce more items in `next_items`, `eof_items`, and `bb_items`.
+///
+/// For more info about the how this happens, see the module-level doc comments and the inline
+/// comments of this function.
+///
+/// # Parameters
+///
+/// - `src`: the current token of the parser.
+/// - `stack`: the "parent" frames of the token tree
+/// - `res`: the match result to store errors
+/// - `cur_items`: the set of current items to be processed. This should be empty by the end of a
+///   successful execution of this function.
+/// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in
+///   the function `parse`.
+/// - `eof_items`: the set of items that would be valid if this was the EOF.
+/// - `bb_items`: the set of items that are waiting for the black-box parser.
+/// - `error_items`: the set of items in errors, used for error-resilient parsing
+fn match_loop_inner<'t>(
+    src: TtIter<'t>,
+    stack: &[TtIter<'t>],
+    res: &mut Match,
+    cur_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    bb_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    next_items: &mut Vec<MatchState<'t>>,
+    eof_items: &mut SmallVec<[MatchState<'t>; 1]>,
+    error_items: &mut SmallVec<[MatchState<'t>; 1]>,
+) {
+    macro_rules! try_push {
+        ($items: expr, $it:expr) => {
+            if $it.is_error {
+                error_items.push($it);
+            } else {
+                $items.push($it);
+            }
+        };
+    }
+
+    while let Some(mut item) = cur_items.pop() {
+        while item.dot.is_eof() {
+            match item.stack.pop() {
+                Some(frame) => {
+                    item.dot = frame;
+                    item.dot.next();
+                }
+                None => break,
+            }
+        }
+        let op = match item.dot.peek() {
+            None => {
+                // We are at or past the end of the matcher of `item`.
+                if item.up.is_some() {
+                    if item.sep_parsed.is_none() {
+                        // Get the `up` matcher
+                        let mut new_pos = *item.up.clone().unwrap();
+                        // Add matches from this repetition to the `matches` of `up`
+                        if let Some(bindings) = new_pos.bindings.last_mut() {
+                            for (i, b) in item.bindings.iter_mut().enumerate() {
+                                bindings.push_nested(i, b.clone()).unwrap();
+                            }
+                        }
+                        // Move the "dot" past the repetition in `up`
+                        new_pos.dot.next();
+                        new_pos.is_error = new_pos.is_error || item.is_error;
+                        cur_items.push(new_pos);
+                    }
+
+                    // Check if we need a separator.
+                    // We check the separator one by one
+                    let sep_idx = *item.sep_parsed.as_ref().unwrap_or(&0);
+                    let sep_len = item.sep.as_ref().map_or(0, Separator::tt_count);
+                    if item.sep.is_some() && sep_idx != sep_len {
+                        let sep = item.sep.as_ref().unwrap();
+                        if src.clone().expect_separator(&sep, sep_idx) {
+                            item.dot.next();
+                            item.sep_parsed = Some(sep_idx + 1);
+                            try_push!(next_items, item);
+                        }
+                    }
+                    // We don't need a separator. Move the "dot" back to the beginning of the matcher
+                    // and try to match again UNLESS we are only allowed to have _one_ repetition.
+                    else if item.sep_kind != Some(RepeatKind::ZeroOrOne) {
+                        item.dot = item.dot.reset();
+                        item.sep_parsed = None;
+                        item.bindings.push(Bindings::default());
+                        cur_items.push(item);
+                    }
+                } else {
+                    // If we are not in a repetition, then being at the end of a matcher means that we have
+                    // reached the potential end of the input.
+                    try_push!(eof_items, item);
+                }
+                continue;
+            }
+            Some(it) => it,
+        };
+
+        // We are in the middle of a matcher.
+        match op {
+            OpDelimited::Op(Op::Repeat { tokens, kind, separator }) => {
+                if matches!(kind, RepeatKind::ZeroOrMore | RepeatKind::ZeroOrOne) {
+                    let mut new_item = item.clone();
+                    new_item.dot.next();
+                    let mut vars = Vec::new();
+                    let bindings = new_item.bindings.last_mut().unwrap();
+                    collect_vars(&mut vars, tokens);
+                    for var in vars {
+                        bindings.push_empty(&var);
+                    }
+                    cur_items.push(new_item);
+                }
+                cur_items.push(MatchState {
+                    dot: tokens.iter_delimited(None),
+                    stack: Default::default(),
+                    up: Some(Box::new(item)),
+                    sep: separator.clone(),
+                    sep_kind: Some(*kind),
+                    sep_parsed: None,
+                    bindings: smallvec![Bindings::default()],
+                    meta_result: None,
+                    is_error: false,
+                })
+            }
+            OpDelimited::Op(Op::Subtree { tokens, delimiter }) => {
+                if let Ok(subtree) = src.clone().expect_subtree() {
+                    if subtree.delimiter_kind() == delimiter.map(|it| it.kind) {
+                        item.stack.push(item.dot);
+                        item.dot = tokens.iter_delimited(delimiter.as_ref());
+                        cur_items.push(item);
+                    }
+                }
+            }
+            OpDelimited::Op(Op::Var { kind, name, .. }) => {
+                if let Some(kind) = kind {
+                    let mut fork = src.clone();
+                    let match_res = match_meta_var(kind.as_str(), &mut fork);
+                    match match_res.err {
+                        None => {
+                            // Some meta variables are optional (e.g. vis)
+                            if match_res.value.is_some() {
+                                item.meta_result = Some((fork, match_res));
+                                try_push!(bb_items, item);
+                            } else {
+                                item.bindings.last_mut().unwrap().push_optional(name);
+                                item.dot.next();
+                                cur_items.push(item);
+                            }
+                        }
+                        Some(err) => {
+                            res.add_err(err);
+                            match match_res.value {
+                                Some(fragment) => {
+                                    item.bindings
+                                        .last_mut()
+                                        .unwrap()
+                                        .inner
+                                        .push((name.clone(), Binding::Fragment(fragment)));
+                                }
+                                _ => {}
+                            }
+                            item.is_error = true;
+                            error_items.push(item);
+                        }
+                    }
+                }
+            }
+            OpDelimited::Op(Op::Leaf(leaf)) => {
+                if let Err(err) = match_leaf(&leaf, &mut src.clone()) {
+                    res.add_err(err);
+                    item.is_error = true;
+                } else {
+                    item.dot.next();
+                }
+                try_push!(next_items, item);
+            }
+            OpDelimited::Open => {
+                if matches!(src.clone().next(), Some(tt::TokenTree::Subtree(..))) {
+                    item.dot.next();
+                    try_push!(next_items, item);
+                }
+            }
+            OpDelimited::Close => {
+                let is_delim_closed = src.peek_n(0).is_none() && !stack.is_empty();
+                if is_delim_closed {
+                    item.dot.next();
+                    try_push!(next_items, item);
+                }
+            }
+        }
+    }
+}
+
+fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
+    let mut src = TtIter::new(src);
+    let mut stack: SmallVec<[TtIter; 1]> = SmallVec::new();
+    let mut res = Match::default();
+    let mut error_reover_item = None;
+
+    let mut cur_items = smallvec![MatchState {
+        dot: pattern.iter_delimited(None),
+        stack: Default::default(),
+        up: None,
+        sep: None,
+        sep_kind: None,
+        sep_parsed: None,
+        bindings: smallvec![Bindings::default()],
+        is_error: false,
+        meta_result: None,
+    }];
+
+    let mut next_items = vec![];
+
+    loop {
+        let mut bb_items = SmallVec::new();
+        let mut eof_items = SmallVec::new();
+        let mut error_items = SmallVec::new();
+
+        stdx::always!(next_items.is_empty());
+
+        match_loop_inner(
+            src.clone(),
+            &stack,
+            &mut res,
+            &mut cur_items,
+            &mut bb_items,
+            &mut next_items,
+            &mut eof_items,
+            &mut error_items,
+        );
+        stdx::always!(cur_items.is_empty());
+
+        if error_items.len() > 0 {
+            error_reover_item = error_items.pop();
+        } else if eof_items.len() > 0 {
+            error_reover_item = Some(eof_items[0].clone());
+        }
+
+        // We need to do some post processing after the `match_loop_inner`.
+        // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
+        // either the parse is ambiguous (which should never happen) or there is a syntax error.
+        if src.peek_n(0).is_none() && stack.is_empty() {
+            if eof_items.len() == 1 {
+                // remove all errors, because it is the correct answer !
+                res = Match::default();
+                res.bindings = eof_items[0].bindings[0].clone();
+            } else {
+                // Error recovery
+                if error_reover_item.is_some() {
+                    res.bindings = error_reover_item.unwrap().bindings[0].clone();
+                }
+                res.add_err(ExpandError::UnexpectedToken);
+            }
+            return res;
+        }
+
+        // If there are no possible next positions AND we aren't waiting for the black-box parser,
+        // then there is a syntax error.
+        //
+        // Another possibility is that we need to call out to parse some rust nonterminal
+        // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong.
+        if (bb_items.is_empty() && next_items.is_empty())
+            || (!bb_items.is_empty() && !next_items.is_empty())
+            || bb_items.len() > 1
+        {
+            res.unmatched_tts += src.len();
+            while let Some(it) = stack.pop() {
+                src = it;
+                res.unmatched_tts += src.len();
+            }
+            res.add_err(err!("leftover tokens"));
+
+            if let Some(mut error_reover_item) = error_reover_item {
+                res.bindings = error_reover_item.bindings.remove(0);
+            }
+            return res;
+        }
+        // Dump all possible `next_items` into `cur_items` for the next iteration.
+        else if !next_items.is_empty() {
+            // Now process the next token
+            cur_items.extend(next_items.drain(..));
+
+            match src.next() {
+                Some(tt::TokenTree::Subtree(subtree)) => {
+                    stack.push(src.clone());
+                    src = TtIter::new(subtree);
+                }
+                None if !stack.is_empty() => src = stack.pop().unwrap(),
+                _ => (),
+            }
+        }
+        // Finally, we have the case where we need to call the black-box parser to get some
+        // nonterminal.
+        else {
+            stdx::always!(bb_items.len() == 1);
+            let mut item = bb_items.pop().unwrap();
+
+            if let Some(OpDelimited::Op(Op::Var { name, .. })) = item.dot.peek() {
+                let (iter, match_res) = item.meta_result.take().unwrap();
+                let bindings = item.bindings.last_mut().unwrap();
+                match match_res.value {
+                    Some(fragment) => {
+                        bindings.inner.push((name.clone(), Binding::Fragment(fragment)));
+                    }
+                    None if match_res.err.is_none() => bindings.push_optional(name),
+                    _ => {}
+                }
+                if let Some(err) = match_res.err {
+                    res.add_err(err);
+                }
+                src = iter.clone();
+                item.dot.next();
+            } else {
+                unreachable!()
+            }
+            cur_items.push(item);
+        }
+        stdx::always!(!cur_items.is_empty());
+    }
+}
+
+fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter) -> Result<(), ExpandError> {
+    let rhs = match src.expect_leaf() {
+        Ok(l) => l,
+        Err(()) => {
+            return Err(err!("expected leaf: `{}`", lhs));
+        }
+    };
+    match (lhs, rhs) {
+        (
+            tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
+            tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
+        ) if lhs == rhs => (),
+        (
+            tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
+            tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
+        ) if lhs == rhs => (),
+        (
+            tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
+            tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
+        ) if lhs == rhs => (),
+        _ => {
+            return Err(ExpandError::UnexpectedToken);
+        }
+    }
+
+    Ok(())
+}
+
+fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragment>> {
+    let fragment = match kind {
+        "path" => Path,
+        "expr" => Expr,
+        "ty" => Type,
+        "pat" => Pattern,
+        "stmt" => Statement,
+        "block" => Block,
+        "meta" => MetaItem,
+        "item" => Item,
+        _ => {
+            let tt_result = match kind {
+                "ident" => input
+                    .expect_ident()
+                    .map(|ident| Some(tt::Leaf::from(ident.clone()).into()))
+                    .map_err(|()| err!("expected ident")),
+                "tt" => input.expect_tt().map(Some).map_err(|()| err!()),
+                "lifetime" => input
+                    .expect_lifetime()
+                    .map(|tt| Some(tt))
+                    .map_err(|()| err!("expected lifetime")),
+                "literal" => {
+                    let neg = input.eat_char('-');
+                    input
+                        .expect_literal()
+                        .map(|literal| {
+                            let lit = tt::Leaf::from(literal.clone());
+                            match neg {
+                                None => Some(lit.into()),
+                                Some(neg) => Some(tt::TokenTree::Subtree(tt::Subtree {
+                                    delimiter: None,
+                                    token_trees: vec![neg, lit.into()],
+                                })),
+                            }
+                        })
+                        .map_err(|()| err!())
+                }
+                // `vis` is optional
+                "vis" => match input.eat_vis() {
+                    Some(vis) => Ok(Some(vis)),
+                    None => Ok(None),
+                },
+                _ => Err(ExpandError::UnexpectedToken),
+            };
+            return tt_result.map(|it| it.map(Fragment::Tokens)).into();
+        }
+    };
+    let result = input.expect_fragment(fragment);
+    result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) })
+}
+
+fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &MetaTemplate) {
+    for op in pattern.iter() {
+        match op {
+            Op::Var { name, .. } => buf.push(name.clone()),
+            Op::Leaf(_) => (),
+            Op::Subtree { tokens, .. } => collect_vars(buf, tokens),
+            Op::Repeat { tokens, .. } => collect_vars(buf, tokens),
+        }
+    }
+}
+
+impl<'a> TtIter<'a> {
+    fn expect_separator(&mut self, separator: &Separator, idx: usize) -> bool {
+        let mut fork = self.clone();
+        let ok = match separator {
+            Separator::Ident(lhs) if idx == 0 => match fork.expect_ident() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
+            },
+            Separator::Literal(lhs) if idx == 0 => match fork.expect_literal() {
+                Ok(rhs) => match rhs {
+                    tt::Leaf::Literal(rhs) => rhs.text == lhs.text,
+                    tt::Leaf::Ident(rhs) => rhs.text == lhs.text,
+                    tt::Leaf::Punct(_) => false,
+                },
+                _ => false,
+            },
+            Separator::Puncts(lhss) if idx < lhss.len() => match fork.expect_punct() {
+                Ok(rhs) => rhs.char == lhss[idx].char,
+                _ => false,
+            },
+            _ => false,
+        };
+        if ok {
+            *self = fork;
+        }
+        ok
+    }
+
+    fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+        match self.peek_n(0) {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
+                return self.expect_lifetime();
+            }
+            _ => (),
+        }
+
+        let tt = self.next().ok_or_else(|| ())?.clone();
+        let punct = match tt {
+            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
+                punct
+            }
+            _ => return Ok(tt),
+        };
+
+        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
+            (
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
+            ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)),
+            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None),
+            _ => return Ok(tt),
+        };
+
+        match (punct.char, second, third) {
+            ('.', '.', Some('.'))
+            | ('.', '.', Some('='))
+            | ('<', '<', Some('='))
+            | ('>', '>', Some('=')) => {
+                let tt2 = self.next().unwrap().clone();
+                let tt3 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into())
+            }
+            ('-', '=', _)
+            | ('-', '>', _)
+            | (':', ':', _)
+            | ('!', '=', _)
+            | ('.', '.', _)
+            | ('*', '=', _)
+            | ('/', '=', _)
+            | ('&', '&', _)
+            | ('&', '=', _)
+            | ('%', '=', _)
+            | ('^', '=', _)
+            | ('+', '=', _)
+            | ('<', '<', _)
+            | ('<', '=', _)
+            | ('=', '=', _)
+            | ('=', '>', _)
+            | ('>', '=', _)
+            | ('>', '>', _)
+            | ('|', '=', _)
+            | ('|', '|', _) => {
+                let tt2 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into())
+            }
+            _ => Ok(tt),
+        }
+    }
+
+    fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+        let punct = self.expect_punct()?;
+        if punct.char != '\'' {
+            return Err(());
+        }
+        let ident = self.expect_ident()?;
+
+        Ok(tt::Subtree {
+            delimiter: None,
+            token_trees: vec![
+                tt::Leaf::Punct(*punct).into(),
+                tt::Leaf::Ident(ident.clone()).into(),
+            ],
+        }
+        .into())
+    }
+
+    fn eat_vis(&mut self) -> Option<tt::TokenTree> {
+        let mut fork = self.clone();
+        match fork.expect_fragment(Visibility) {
+            ExpandResult { value: tt, err: None } => {
+                *self = fork;
+                tt
+            }
+            ExpandResult { value: _, err: Some(_) } => None,
+        }
+    }
+
+    fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
+        let mut fork = self.clone();
+        match fork.expect_char(c) {
+            Ok(_) => {
+                let tt = self.next().cloned();
+                *self = fork;
+                tt
+            }
+            Err(_) => None,
+        }
+    }
+}
diff --git a/crates/mbe/src/mbe_expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs
index 59a3c80a8..ad9953a7d 100644
--- a/crates/mbe/src/mbe_expander/transcriber.rs
+++ b/crates/mbe/src/expander/transcriber.rs
@@ -2,23 +2,28 @@
 //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
 
 use syntax::SmolStr;
+use tt::Delimiter;
 
 use super::ExpandResult;
 use crate::{
-    mbe_expander::{Binding, Bindings, Fragment},
+    expander::{Binding, Bindings, Fragment},
     parser::{Op, RepeatKind, Separator},
     ExpandError, MetaTemplate,
 };
 
 impl Bindings {
     fn contains(&self, name: &str) -> bool {
-        self.inner.contains_key(name)
+        self.inner.iter().any(|(n, _)| n == name)
     }
 
     fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> {
-        let mut b = self.inner.get(name).ok_or_else(|| {
-            ExpandError::BindingError(format!("could not find binding `{}`", name))
-        })?;
+        let mut b: &Binding = self
+            .inner
+            .iter()
+            .find_map(|(n, b)| if n == name { Some(b) } else { None })
+            .ok_or_else(|| {
+                ExpandError::BindingError(format!("could not find binding `{}`", name))
+            })?;
         for nesting_state in nesting.iter_mut() {
             nesting_state.hit = true;
             b = match b {
@@ -54,10 +59,9 @@ pub(super) fn transcribe(
     template: &MetaTemplate,
     bindings: &Bindings,
 ) -> ExpandResult<tt::Subtree> {
-    assert!(template.delimiter == None);
     let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() };
     let mut arena: Vec<tt::TokenTree> = Vec::new();
-    expand_subtree(&mut ctx, template, &mut arena)
+    expand_subtree(&mut ctx, template, None, &mut arena)
 }
 
 #[derive(Debug)]
@@ -80,23 +84,18 @@ struct ExpandCtx<'a> {
 fn expand_subtree(
     ctx: &mut ExpandCtx,
     template: &MetaTemplate,
+    delimiter: Option<Delimiter>,
     arena: &mut Vec<tt::TokenTree>,
 ) -> ExpandResult<tt::Subtree> {
     // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
     let start_elements = arena.len();
     let mut err = None;
     for op in template.iter() {
-        let op = match op {
-            Ok(op) => op,
-            Err(e) => {
-                err = Some(e.clone());
-                break;
-            }
-        };
         match op {
             Op::Leaf(tt) => arena.push(tt.clone().into()),
-            Op::Subtree(tt) => {
-                let ExpandResult { value: tt, err: e } = expand_subtree(ctx, &tt, arena);
+            Op::Subtree { tokens, delimiter } => {
+                let ExpandResult { value: tt, err: e } =
+                    expand_subtree(ctx, &tokens, *delimiter, arena);
                 err = err.or(e);
                 arena.push(tt.into());
             }
@@ -105,7 +104,7 @@ fn expand_subtree(
                 err = err.or(e);
                 push_fragment(arena, fragment);
             }
-            Op::Repeat { subtree, kind, separator } => {
+            Op::Repeat { tokens: subtree, kind, separator } => {
                 let ExpandResult { value: fragment, err: e } =
                     expand_repeat(ctx, subtree, *kind, separator, arena);
                 err = err.or(e);
@@ -115,7 +114,7 @@ fn expand_subtree(
     }
     // drain the elements added in this instance of expand_subtree
     let tts = arena.drain(start_elements..arena.len()).collect();
-    ExpandResult { value: tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err }
+    ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err }
 }
 
 fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
@@ -169,7 +168,7 @@ fn expand_repeat(
     let mut counter = 0;
 
     loop {
-        let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, arena);
+        let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, None, arena);
         let nesting_state = ctx.nesting.last_mut().unwrap();
         if nesting_state.at_end || !nesting_state.hit {
             break;
diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
index 19543d777..f3d2da55a 100644
--- a/crates/mbe/src/lib.rs
+++ b/crates/mbe/src/lib.rs
@@ -4,7 +4,7 @@
 //! `TokenTree`s as well!
 
 mod parser;
-mod mbe_expander;
+mod expander;
 mod syntax_bridge;
 mod tt_iter;
 mod subtree_source;
@@ -12,18 +12,24 @@ mod subtree_source;
 #[cfg(test)]
 mod tests;
 
+#[cfg(test)]
+mod benchmark;
+
 use std::fmt;
 
+use test_utils::mark;
 pub use tt::{Delimiter, DelimiterKind, Punct};
 
 use crate::{
-    parser::{parse_pattern, parse_template, Op},
+    parser::{parse_pattern, parse_template, MetaTemplate, Op},
     tt_iter::TtIter,
 };
 
 #[derive(Debug, PartialEq, Eq)]
 pub enum ParseError {
+    UnexpectedToken(String),
     Expected(String),
+    InvalidRepeat,
     RepetitionEmptyTokenTree,
 }
 
@@ -33,7 +39,6 @@ pub enum ExpandError {
     UnexpectedToken,
     BindingError(String),
     ConversionError,
-    InvalidRepeat,
     ProcMacroError(tt::ExpansionError),
     UnresolvedProcMacro,
     Other(String),
@@ -52,7 +57,6 @@ impl fmt::Display for ExpandError {
             ExpandError::UnexpectedToken => f.write_str("unexpected token in input"),
             ExpandError::BindingError(e) => f.write_str(e),
             ExpandError::ConversionError => f.write_str("could not convert tokens"),
-            ExpandError::InvalidRepeat => f.write_str("invalid repeat expression"),
             ExpandError::ProcMacroError(e) => e.fmt(f),
             ExpandError::UnresolvedProcMacro => f.write_str("unresolved proc macro"),
             ExpandError::Other(e) => f.write_str(e),
@@ -61,8 +65,8 @@ impl fmt::Display for ExpandError {
 }
 
 pub use crate::syntax_bridge::{
-    ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node,
-    TokenMap,
+    ast_to_token_tree, parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree,
+    token_tree_to_syntax_node, TokenMap,
 };
 
 /// This struct contains AST for a single `macro_rules` definition. What might
@@ -76,26 +80,18 @@ pub struct MacroRules {
     shift: Shift,
 }
 
+/// For Macro 2.0
 #[derive(Clone, Debug, PartialEq, Eq)]
-struct Rule {
-    lhs: MetaTemplate,
-    rhs: MetaTemplate,
+pub struct MacroDef {
+    rules: Vec<Rule>,
+    /// Highest id of the token we have in TokenMap
+    shift: Shift,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
-struct MetaTemplate {
-    delimiter: Option<Delimiter>,
-    tokens: Vec<Result<Op, ExpandError>>,
-}
-
-impl<'a> MetaTemplate {
-    fn iter(&self) -> impl Iterator<Item = &Result<Op, ExpandError>> {
-        self.tokens.iter()
-    }
-
-    fn delimiter_kind(&self) -> Option<DelimiterKind> {
-        self.delimiter.map(|it| it.kind)
-    }
+struct Rule {
+    lhs: MetaTemplate,
+    rhs: MetaTemplate,
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -179,7 +175,7 @@ impl MacroRules {
         let mut src = TtIter::new(tt);
         let mut rules = Vec::new();
         while src.len() > 0 {
-            let rule = Rule::parse(&mut src)?;
+            let rule = Rule::parse(&mut src, true)?;
             rules.push(rule);
             if let Err(()) = src.expect_char(';') {
                 if src.len() > 0 {
@@ -200,7 +196,58 @@ impl MacroRules {
         // apply shift
         let mut tt = tt.clone();
         self.shift.shift_all(&mut tt);
-        mbe_expander::expand(self, &tt)
+        expander::expand_rules(&self.rules, &tt)
+    }
+
+    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
+        self.shift.shift(id)
+    }
+
+    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) {
+        match self.shift.unshift(id) {
+            Some(id) => (id, Origin::Call),
+            None => (id, Origin::Def),
+        }
+    }
+}
+
+impl MacroDef {
+    pub fn parse(tt: &tt::Subtree) -> Result<MacroDef, ParseError> {
+        let mut src = TtIter::new(tt);
+        let mut rules = Vec::new();
+
+        if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() {
+            mark::hit!(parse_macro_def_rules);
+            while src.len() > 0 {
+                let rule = Rule::parse(&mut src, true)?;
+                rules.push(rule);
+                if let Err(()) = src.expect_char(';') {
+                    if src.len() > 0 {
+                        return Err(ParseError::Expected("expected `;`".to_string()));
+                    }
+                    break;
+                }
+            }
+        } else {
+            mark::hit!(parse_macro_def_simple);
+            let rule = Rule::parse(&mut src, false)?;
+            if src.len() != 0 {
+                return Err(ParseError::Expected("remain tokens in macro def".to_string()));
+            }
+            rules.push(rule);
+        }
+        for rule in rules.iter() {
+            validate(&rule.lhs)?;
+        }
+
+        Ok(MacroDef { rules, shift: Shift::new(tt) })
+    }
+
+    pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+        // apply shift
+        let mut tt = tt.clone();
+        self.shift.shift_all(&mut tt);
+        expander::expand_rules(&self.rules, &tt)
     }
 
     pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
@@ -216,57 +263,50 @@ impl MacroRules {
 }
 
 impl Rule {
-    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
+    fn parse(src: &mut TtIter, expect_arrow: bool) -> Result<Rule, ParseError> {
         let lhs = src
             .expect_subtree()
             .map_err(|()| ParseError::Expected("expected subtree".to_string()))?;
-        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
-        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
+        if expect_arrow {
+            src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
+            src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
+        }
         let rhs = src
             .expect_subtree()
             .map_err(|()| ParseError::Expected("expected subtree".to_string()))?;
 
-        let lhs = MetaTemplate { tokens: parse_pattern(&lhs), delimiter: None };
-        let rhs = MetaTemplate { tokens: parse_template(&rhs), delimiter: None };
+        let lhs = MetaTemplate(parse_pattern(&lhs)?);
+        let rhs = MetaTemplate(parse_template(&rhs)?);
 
         Ok(crate::Rule { lhs, rhs })
     }
 }
 
-fn to_parse_error(e: &ExpandError) -> ParseError {
-    let msg = match e {
-        ExpandError::InvalidRepeat => "invalid repeat".to_string(),
-        _ => "invalid macro definition".to_string(),
-    };
-    ParseError::Expected(msg)
-}
-
 fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
     for op in pattern.iter() {
-        let op = op.as_ref().map_err(|e| to_parse_error(&e))?;
-
         match op {
-            Op::Subtree(subtree) => validate(&subtree)?,
-            Op::Repeat { subtree, separator, .. } => {
+            Op::Subtree { tokens, .. } => validate(&tokens)?,
+            Op::Repeat { tokens: subtree, separator, .. } => {
                 // Checks that no repetition which could match an empty token
                 // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558
 
                 if separator.is_none() {
                     if subtree.iter().all(|child_op| {
-                        match child_op.as_ref().map_err(to_parse_error) {
-                            Ok(Op::Var { kind, .. }) => {
+                        match child_op {
+                            Op::Var { kind, .. } => {
                                 // vis is optional
                                 if kind.as_ref().map_or(false, |it| it == "vis") {
                                     return true;
                                 }
                             }
-                            Ok(Op::Repeat { kind, .. }) => {
+                            Op::Repeat { kind, .. } => {
                                 return matches!(
                                     kind,
                                     parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne
                                 )
                             }
-                            _ => {}
+                            Op::Leaf(_) => {}
+                            Op::Subtree { .. } => {}
                         }
                         false
                     }) {
diff --git a/crates/mbe/src/mbe_expander/matcher.rs b/crates/mbe/src/mbe_expander/matcher.rs
deleted file mode 100644
index d32e60521..000000000
--- a/crates/mbe/src/mbe_expander/matcher.rs
+++ /dev/null
@@ -1,502 +0,0 @@
-//! FIXME: write short doc here
-
-use crate::{
-    mbe_expander::{Binding, Bindings, Fragment},
-    parser::{Op, RepeatKind, Separator},
-    subtree_source::SubtreeTokenSource,
-    tt_iter::TtIter,
-    ExpandError, MetaTemplate,
-};
-
-use super::ExpandResult;
-use parser::{FragmentKind::*, TreeSink};
-use syntax::{SmolStr, SyntaxKind};
-use tt::buffer::{Cursor, TokenBuffer};
-
-impl Bindings {
-    fn push_optional(&mut self, name: &SmolStr) {
-        // FIXME: Do we have a better way to represent an empty token ?
-        // Insert an empty subtree for empty token
-        let tt = tt::Subtree::default().into();
-        self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
-    }
-
-    fn push_empty(&mut self, name: &SmolStr) {
-        self.inner.insert(name.clone(), Binding::Empty);
-    }
-
-    fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
-        for (key, value) in nested.inner {
-            if !self.inner.contains_key(&key) {
-                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
-            }
-            match self.inner.get_mut(&key) {
-                Some(Binding::Nested(it)) => {
-                    // insert empty nested bindings before this one
-                    while it.len() < idx {
-                        it.push(Binding::Nested(vec![]));
-                    }
-                    it.push(value);
-                }
-                _ => {
-                    return Err(ExpandError::BindingError(format!(
-                        "could not find binding `{}`",
-                        key
-                    )));
-                }
-            }
-        }
-        Ok(())
-    }
-}
-
-macro_rules! err {
-    () => {
-        ExpandError::BindingError(format!(""))
-    };
-    ($($tt:tt)*) => {
-        ExpandError::BindingError(format!($($tt)*))
-    };
-}
-
-#[derive(Debug, Default)]
-pub(super) struct Match {
-    pub(super) bindings: Bindings,
-    /// We currently just keep the first error and count the rest to compare matches.
-    pub(super) err: Option<ExpandError>,
-    pub(super) err_count: usize,
-    /// How many top-level token trees were left to match.
-    pub(super) unmatched_tts: usize,
-}
-
-impl Match {
-    pub(super) fn add_err(&mut self, err: ExpandError) {
-        let prev_err = self.err.take();
-        self.err = prev_err.or(Some(err));
-        self.err_count += 1;
-    }
-}
-
-// General note: These functions have two channels to return errors, a `Result`
-// return value and the `&mut Match`. The returned Result is for pattern parsing
-// errors; if a branch of the macro definition doesn't parse, it doesn't make
-// sense to try using it. Matching errors are added to the `Match`. It might
-// make sense to make pattern parsing a separate step?
-
-pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Result<Match, ExpandError> {
-    assert!(pattern.delimiter == None);
-
-    let mut res = Match::default();
-    let mut src = TtIter::new(src);
-
-    match_subtree(&mut res, pattern, &mut src)?;
-
-    if src.len() > 0 {
-        res.unmatched_tts += src.len();
-        res.add_err(err!("leftover tokens"));
-    }
-
-    Ok(res)
-}
-
-fn match_subtree(
-    res: &mut Match,
-    pattern: &MetaTemplate,
-    src: &mut TtIter,
-) -> Result<(), ExpandError> {
-    for op in pattern.iter() {
-        match op.as_ref().map_err(|err| err.clone())? {
-            Op::Leaf(lhs) => {
-                let rhs = match src.expect_leaf() {
-                    Ok(l) => l,
-                    Err(()) => {
-                        res.add_err(err!("expected leaf: `{}`", lhs));
-                        continue;
-                    }
-                };
-                match (lhs, rhs) {
-                    (
-                        tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
-                        tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
-                    ) if lhs == rhs => (),
-                    (
-                        tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
-                        tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
-                    ) if lhs == rhs => (),
-                    (
-                        tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
-                        tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
-                    ) if lhs == rhs => (),
-                    _ => {
-                        res.add_err(ExpandError::UnexpectedToken);
-                    }
-                }
-            }
-            Op::Subtree(lhs) => {
-                let rhs = match src.expect_subtree() {
-                    Ok(s) => s,
-                    Err(()) => {
-                        res.add_err(err!("expected subtree"));
-                        continue;
-                    }
-                };
-                if lhs.delimiter_kind() != rhs.delimiter_kind() {
-                    res.add_err(err!("mismatched delimiter"));
-                    continue;
-                }
-                let mut src = TtIter::new(rhs);
-                match_subtree(res, lhs, &mut src)?;
-                if src.len() > 0 {
-                    res.add_err(err!("leftover tokens"));
-                }
-            }
-            Op::Var { name, kind, .. } => {
-                let kind = match kind {
-                    Some(k) => k,
-                    None => {
-                        res.add_err(ExpandError::UnexpectedToken);
-                        continue;
-                    }
-                };
-                let ExpandResult { value: matched, err: match_err } =
-                    match_meta_var(kind.as_str(), src);
-                match matched {
-                    Some(fragment) => {
-                        res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
-                    }
-                    None if match_err.is_none() => res.bindings.push_optional(name),
-                    _ => {}
-                }
-                if let Some(err) = match_err {
-                    res.add_err(err);
-                }
-            }
-            Op::Repeat { subtree, kind, separator } => {
-                match_repeat(res, subtree, *kind, separator, src)?;
-            }
-        }
-    }
-    Ok(())
-}
-
-impl<'a> TtIter<'a> {
-    fn eat_separator(&mut self, separator: &Separator) -> bool {
-        let mut fork = self.clone();
-        let ok = match separator {
-            Separator::Ident(lhs) => match fork.expect_ident() {
-                Ok(rhs) => rhs.text == lhs.text,
-                _ => false,
-            },
-            Separator::Literal(lhs) => match fork.expect_literal() {
-                Ok(rhs) => match rhs {
-                    tt::Leaf::Literal(rhs) => rhs.text == lhs.text,
-                    tt::Leaf::Ident(rhs) => rhs.text == lhs.text,
-                    tt::Leaf::Punct(_) => false,
-                },
-                _ => false,
-            },
-            Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
-                Ok(rhs) => rhs.char == lhs.char,
-                _ => false,
-            }),
-        };
-        if ok {
-            *self = fork;
-        }
-        ok
-    }
-
-    pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
-        match self.peek_n(0) {
-            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
-                return self.expect_lifetime();
-            }
-            _ => (),
-        }
-
-        let tt = self.next().ok_or_else(|| ())?.clone();
-        let punct = match tt {
-            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
-                punct
-            }
-            _ => return Ok(tt),
-        };
-
-        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
-            (
-                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
-                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
-            ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)),
-            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None),
-            _ => return Ok(tt),
-        };
-
-        match (punct.char, second, third) {
-            ('.', '.', Some('.'))
-            | ('.', '.', Some('='))
-            | ('<', '<', Some('='))
-            | ('>', '>', Some('=')) => {
-                let tt2 = self.next().unwrap().clone();
-                let tt3 = self.next().unwrap().clone();
-                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into())
-            }
-            ('-', '=', _)
-            | ('-', '>', _)
-            | (':', ':', _)
-            | ('!', '=', _)
-            | ('.', '.', _)
-            | ('*', '=', _)
-            | ('/', '=', _)
-            | ('&', '&', _)
-            | ('&', '=', _)
-            | ('%', '=', _)
-            | ('^', '=', _)
-            | ('+', '=', _)
-            | ('<', '<', _)
-            | ('<', '=', _)
-            | ('=', '=', _)
-            | ('=', '>', _)
-            | ('>', '=', _)
-            | ('>', '>', _)
-            | ('|', '=', _)
-            | ('|', '|', _) => {
-                let tt2 = self.next().unwrap().clone();
-                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into())
-            }
-            _ => Ok(tt),
-        }
-    }
-
-    pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
-        let punct = self.expect_punct()?;
-        if punct.char != '\'' {
-            return Err(());
-        }
-        let ident = self.expect_ident()?;
-
-        Ok(tt::Subtree {
-            delimiter: None,
-            token_trees: vec![
-                tt::Leaf::Punct(*punct).into(),
-                tt::Leaf::Ident(ident.clone()).into(),
-            ],
-        }
-        .into())
-    }
-
-    pub(crate) fn expect_fragment(
-        &mut self,
-        fragment_kind: parser::FragmentKind,
-    ) -> ExpandResult<Option<tt::TokenTree>> {
-        pub(crate) struct OffsetTokenSink<'a> {
-            pub(crate) cursor: Cursor<'a>,
-            pub(crate) error: bool,
-        }
-
-        impl<'a> TreeSink for OffsetTokenSink<'a> {
-            fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
-                if kind == SyntaxKind::LIFETIME_IDENT {
-                    n_tokens = 2;
-                }
-                for _ in 0..n_tokens {
-                    self.cursor = self.cursor.bump_subtree();
-                }
-            }
-            fn start_node(&mut self, _kind: SyntaxKind) {}
-            fn finish_node(&mut self) {}
-            fn error(&mut self, _error: parser::ParseError) {
-                self.error = true;
-            }
-        }
-
-        let buffer = TokenBuffer::from_tokens(&self.inner.as_slice());
-        let mut src = SubtreeTokenSource::new(&buffer);
-        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
-
-        parser::parse_fragment(&mut src, &mut sink, fragment_kind);
-
-        let mut err = None;
-        if !sink.cursor.is_root() || sink.error {
-            err = Some(err!("expected {:?}", fragment_kind));
-        }
-
-        let mut curr = buffer.begin();
-        let mut res = vec![];
-
-        if sink.cursor.is_root() {
-            while curr != sink.cursor {
-                if let Some(token) = curr.token_tree() {
-                    res.push(token);
-                }
-                curr = curr.bump();
-            }
-        }
-        self.inner = self.inner.as_slice()[res.len()..].iter();
-        if res.len() == 0 && err.is_none() {
-            err = Some(err!("no tokens consumed"));
-        }
-        let res = match res.len() {
-            1 => Some(res[0].cloned()),
-            0 => None,
-            _ => Some(tt::TokenTree::Subtree(tt::Subtree {
-                delimiter: None,
-                token_trees: res.into_iter().map(|it| it.cloned()).collect(),
-            })),
-        };
-        ExpandResult { value: res, err }
-    }
-
-    pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
-        let mut fork = self.clone();
-        match fork.expect_fragment(Visibility) {
-            ExpandResult { value: tt, err: None } => {
-                *self = fork;
-                tt
-            }
-            ExpandResult { value: _, err: Some(_) } => None,
-        }
-    }
-
-    pub(crate) fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
-        let mut fork = self.clone();
-        match fork.expect_char(c) {
-            Ok(_) => {
-                let tt = self.next().cloned();
-                *self = fork;
-                tt
-            }
-            Err(_) => None,
-        }
-    }
-}
-
-pub(super) fn match_repeat(
-    res: &mut Match,
-    pattern: &MetaTemplate,
-    kind: RepeatKind,
-    separator: &Option<Separator>,
-    src: &mut TtIter,
-) -> Result<(), ExpandError> {
-    // Dirty hack to make macro-expansion terminate.
-    // This should be replaced by a proper macro-by-example implementation
-    let mut limit = 65536;
-    let mut counter = 0;
-
-    for i in 0.. {
-        let mut fork = src.clone();
-
-        if let Some(separator) = &separator {
-            if i != 0 && !fork.eat_separator(separator) {
-                break;
-            }
-        }
-
-        let mut nested = Match::default();
-        match_subtree(&mut nested, pattern, &mut fork)?;
-        if nested.err.is_none() {
-            limit -= 1;
-            if limit == 0 {
-                log::warn!(
-                    "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}",
-                    pattern,
-                    src,
-                    kind,
-                    separator
-                );
-                break;
-            }
-            *src = fork;
-
-            if let Err(err) = res.bindings.push_nested(counter, nested.bindings) {
-                res.add_err(err);
-            }
-            counter += 1;
-            if counter == 1 {
-                if let RepeatKind::ZeroOrOne = kind {
-                    break;
-                }
-            }
-        } else {
-            break;
-        }
-    }
-
-    match (kind, counter) {
-        (RepeatKind::OneOrMore, 0) => {
-            res.add_err(ExpandError::UnexpectedToken);
-        }
-        (_, 0) => {
-            // Collect all empty variables in subtrees
-            let mut vars = Vec::new();
-            collect_vars(&mut vars, pattern)?;
-            for var in vars {
-                res.bindings.push_empty(&var)
-            }
-        }
-        _ => (),
-    }
-    Ok(())
-}
-
-fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragment>> {
-    let fragment = match kind {
-        "path" => Path,
-        "expr" => Expr,
-        "ty" => Type,
-        "pat" => Pattern,
-        "stmt" => Statement,
-        "block" => Block,
-        "meta" => MetaItem,
-        "item" => Item,
-        _ => {
-            let tt_result = match kind {
-                "ident" => input
-                    .expect_ident()
-                    .map(|ident| Some(tt::Leaf::from(ident.clone()).into()))
-                    .map_err(|()| err!("expected ident")),
-                "tt" => input.expect_tt().map(Some).map_err(|()| err!()),
-                "lifetime" => input
-                    .expect_lifetime()
-                    .map(|tt| Some(tt))
-                    .map_err(|()| err!("expected lifetime")),
-                "literal" => {
-                    let neg = input.eat_char('-');
-                    input
-                        .expect_literal()
-                        .map(|literal| {
-                            let lit = tt::Leaf::from(literal.clone());
-                            match neg {
-                                None => Some(lit.into()),
-                                Some(neg) => Some(tt::TokenTree::Subtree(tt::Subtree {
-                                    delimiter: None,
-                                    token_trees: vec![neg, lit.into()],
-                                })),
-                            }
-                        })
-                        .map_err(|()| err!())
-                }
-                // `vis` is optional
-                "vis" => match input.eat_vis() {
-                    Some(vis) => Ok(Some(vis)),
-                    None => Ok(None),
-                },
-                _ => Err(ExpandError::UnexpectedToken),
-            };
-            return tt_result.map(|it| it.map(Fragment::Tokens)).into();
-        }
-    };
-    let result = input.expect_fragment(fragment);
-    result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) })
-}
-
-fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &MetaTemplate) -> Result<(), ExpandError> {
-    for op in pattern.iter() {
-        match op.as_ref().map_err(|e| e.clone())? {
-            Op::Var { name, .. } => buf.push(name.clone()),
-            Op::Leaf(_) => (),
-            Op::Subtree(subtree) => collect_vars(buf, subtree)?,
-            Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
-        }
-    }
-    Ok(())
-}
diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs
index f3047972d..8671322e1 100644
--- a/crates/mbe/src/parser.rs
+++ b/crates/mbe/src/parser.rs
@@ -3,15 +3,84 @@
 
 use smallvec::SmallVec;
 use syntax::SmolStr;
+use tt::Delimiter;
 
-use crate::{tt_iter::TtIter, ExpandError, MetaTemplate};
+use crate::{tt_iter::TtIter, ParseError};
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) struct MetaTemplate(pub(crate) Vec<Op>);
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum OpDelimited<'a> {
+    Op(&'a Op),
+    Open,
+    Close,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct OpDelimitedIter<'a> {
+    inner: &'a Vec<Op>,
+    delimited: Option<&'a Delimiter>,
+    idx: usize,
+}
+
+impl<'a> OpDelimitedIter<'a> {
+    pub(crate) fn is_eof(&self) -> bool {
+        let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 };
+        self.idx >= len
+    }
+
+    pub(crate) fn peek(&self) -> Option<OpDelimited<'a>> {
+        match self.delimited {
+            None => self.inner.get(self.idx).map(OpDelimited::Op),
+            Some(_) => match self.idx {
+                0 => Some(OpDelimited::Open),
+                i if i == self.inner.len() + 1 => Some(OpDelimited::Close),
+                i => self.inner.get(i - 1).map(OpDelimited::Op),
+            },
+        }
+    }
+
+    pub(crate) fn reset(&self) -> Self {
+        Self { inner: &self.inner, idx: 0, delimited: self.delimited }
+    }
+}
+
+impl<'a> Iterator for OpDelimitedIter<'a> {
+    type Item = OpDelimited<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let res = self.peek();
+        self.idx += 1;
+        res
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let len = self.inner.len() + if self.delimited.is_some() { 2 } else { 0 };
+        let remain = len.checked_sub(self.idx).unwrap_or(0);
+        (remain, Some(remain))
+    }
+}
+
+impl<'a> MetaTemplate {
+    pub(crate) fn iter(&self) -> impl Iterator<Item = &Op> {
+        self.0.iter()
+    }
+
+    pub(crate) fn iter_delimited(
+        &'a self,
+        delimited: Option<&'a Delimiter>,
+    ) -> OpDelimitedIter<'a> {
+        OpDelimitedIter { inner: &self.0, idx: 0, delimited }
+    }
+}
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(crate) enum Op {
     Var { name: SmolStr, kind: Option<SmolStr>, id: tt::TokenId },
-    Repeat { subtree: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
+    Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
     Leaf(tt::Leaf),
-    Subtree(MetaTemplate),
+    Subtree { tokens: MetaTemplate, delimiter: Option<Delimiter> },
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -46,12 +115,22 @@ impl PartialEq for Separator {
     }
 }
 
-pub(crate) fn parse_template(template: &tt::Subtree) -> Vec<Result<Op, ExpandError>> {
-    parse_inner(&template, Mode::Template)
+impl Separator {
+    pub(crate) fn tt_count(&self) -> usize {
+        match self {
+            Separator::Literal(_) => 1,
+            Separator::Ident(_) => 1,
+            Separator::Puncts(it) => it.len(),
+        }
+    }
+}
+
+pub(crate) fn parse_template(template: &tt::Subtree) -> Result<Vec<Op>, ParseError> {
+    parse_inner(&template, Mode::Template).into_iter().collect()
 }
 
-pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Vec<Result<Op, ExpandError>> {
-    parse_inner(&pattern, Mode::Pattern)
+pub(crate) fn parse_pattern(pattern: &tt::Subtree) -> Result<Vec<Op>, ParseError> {
+    parse_inner(&pattern, Mode::Pattern).into_iter().collect()
 }
 
 #[derive(Clone, Copy)]
@@ -60,7 +139,7 @@ enum Mode {
     Template,
 }
 
-fn parse_inner(tt: &tt::Subtree, mode: Mode) -> Vec<Result<Op, ExpandError>> {
+fn parse_inner(tt: &tt::Subtree, mode: Mode) -> Vec<Result<Op, ParseError>> {
     let mut src = TtIter::new(&tt);
     std::iter::from_fn(move || {
         let first = src.next()?;
@@ -71,7 +150,7 @@ fn parse_inner(tt: &tt::Subtree, mode: Mode) -> Vec<Result<Op, ExpandError>> {
 
 macro_rules! err {
     ($($tt:tt)*) => {
-        ExpandError::UnexpectedToken
+        ParseError::UnexpectedToken(($($tt)*).to_string())
     };
 }
 
@@ -81,7 +160,7 @@ macro_rules! bail {
     };
 }
 
-fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Result<Op, ExpandError> {
+fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Result<Op, ParseError> {
     let res = match first {
         tt::TokenTree::Leaf(leaf @ tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
             // Note that the '$' itself is a valid token inside macro_rules.
@@ -92,17 +171,17 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
             match second {
                 tt::TokenTree::Subtree(subtree) => {
                     let (separator, kind) = parse_repeat(src)?;
-                    let delimiter = subtree.delimiter;
-                    let tokens = parse_inner(&subtree, mode);
-                    let subtree = MetaTemplate { tokens, delimiter };
-                    Op::Repeat { subtree, separator, kind }
+                    let tokens = parse_inner(&subtree, mode)
+                        .into_iter()
+                        .collect::<Result<Vec<Op>, ParseError>>()?;
+                    Op::Repeat { tokens: MetaTemplate(tokens), separator, kind }
                 }
                 tt::TokenTree::Leaf(leaf) => match leaf {
                     tt::Leaf::Punct(punct) => {
                         static UNDERSCORE: SmolStr = SmolStr::new_inline("_");
 
                         if punct.char != '_' {
-                            return Err(ExpandError::UnexpectedToken);
+                            return Err(ParseError::Expected("_".to_string()));
                         }
                         let name = UNDERSCORE.clone();
                         let kind = eat_fragment_kind(src, mode)?;
@@ -134,16 +213,15 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
         }
         tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()),
         tt::TokenTree::Subtree(subtree) => {
-            let delimiter = subtree.delimiter;
-            let tokens = parse_inner(&subtree, mode);
-            let subtree = MetaTemplate { tokens, delimiter };
-            Op::Subtree(subtree)
+            let tokens =
+                parse_inner(&subtree, mode).into_iter().collect::<Result<Vec<Op>, ParseError>>()?;
+            Op::Subtree { tokens: MetaTemplate(tokens), delimiter: subtree.delimiter }
         }
     };
     Ok(res)
 }
 
-fn eat_fragment_kind<'a>(src: &mut TtIter<'a>, mode: Mode) -> Result<Option<SmolStr>, ExpandError> {
+fn eat_fragment_kind<'a>(src: &mut TtIter<'a>, mode: Mode) -> Result<Option<SmolStr>, ParseError> {
     if let Mode::Pattern = mode {
         src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?;
         let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?;
@@ -156,12 +234,12 @@ fn is_boolean_literal(lit: &tt::Literal) -> bool {
     matches!(lit.text.as_str(), "true" | "false")
 }
 
-fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> {
+fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ParseError> {
     let mut separator = Separator::Puncts(SmallVec::new());
     for tt in src {
         let tt = match tt {
             tt::TokenTree::Leaf(leaf) => leaf,
-            tt::TokenTree::Subtree(_) => return Err(ExpandError::InvalidRepeat),
+            tt::TokenTree::Subtree(_) => return Err(ParseError::InvalidRepeat),
         };
         let has_sep = match &separator {
             Separator::Puncts(puncts) => !puncts.is_empty(),
@@ -169,7 +247,7 @@ fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), Exp
         };
         match tt {
             tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => {
-                return Err(ExpandError::InvalidRepeat)
+                return Err(ParseError::InvalidRepeat)
             }
             tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()),
             tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
@@ -182,11 +260,11 @@ fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), Exp
                         match &mut separator {
                             Separator::Puncts(puncts) => {
                                 if puncts.len() == 3 {
-                                    return Err(ExpandError::InvalidRepeat);
+                                    return Err(ParseError::InvalidRepeat);
                                 }
                                 puncts.push(punct.clone())
                             }
-                            _ => return Err(ExpandError::InvalidRepeat),
+                            _ => return Err(ParseError::InvalidRepeat),
                         }
                         continue;
                     }
@@ -196,5 +274,5 @@ fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), Exp
             }
         }
     }
-    Err(ExpandError::InvalidRepeat)
+    Err(ParseError::InvalidRepeat)
 }
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
index 2a41d8167..aacae1026 100644
--- a/crates/mbe/src/syntax_bridge.rs
+++ b/crates/mbe/src/syntax_bridge.rs
@@ -10,22 +10,24 @@ use syntax::{
 };
 use tt::buffer::{Cursor, TokenBuffer};
 
-use crate::subtree_source::SubtreeTokenSource;
 use crate::ExpandError;
+use crate::{subtree_source::SubtreeTokenSource, tt_iter::TtIter};
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum TokenTextRange {
     Token(TextRange),
-    Delimiter(TextRange, TextRange),
+    Delimiter(TextRange),
 }
 
 impl TokenTextRange {
     pub fn by_kind(self, kind: SyntaxKind) -> Option<TextRange> {
         match self {
             TokenTextRange::Token(it) => Some(it),
-            TokenTextRange::Delimiter(open, close) => match kind {
-                T!['{'] | T!['('] | T!['['] => Some(open),
-                T!['}'] | T![')'] | T![']'] => Some(close),
+            TokenTextRange::Delimiter(it) => match kind {
+                T!['{'] | T!['('] | T!['['] => Some(TextRange::at(it.start(), 1.into())),
+                T!['}'] | T![')'] | T![']'] => {
+                    Some(TextRange::at(it.end() - TextSize::of('}'), 1.into()))
+                }
                 _ => None,
             },
         }
@@ -110,12 +112,51 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
     Some((subtree, conv.id_alloc.map))
 }
 
+/// Split token tree with seperate expr: $($e:expr)SEP*
+pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> {
+    if tt.token_trees.is_empty() {
+        return Vec::new();
+    }
+
+    let mut iter = TtIter::new(tt);
+    let mut res = Vec::new();
+
+    while iter.peek_n(0).is_some() {
+        let expanded = iter.expect_fragment(FragmentKind::Expr);
+        if expanded.err.is_some() {
+            break;
+        }
+
+        res.push(match expanded.value {
+            None => break,
+            Some(tt @ tt::TokenTree::Leaf(_)) => {
+                tt::Subtree { delimiter: None, token_trees: vec![tt.into()] }
+            }
+            Some(tt::TokenTree::Subtree(tt)) => tt,
+        });
+
+        let mut fork = iter.clone();
+        if fork.expect_char(sep).is_err() {
+            break;
+        }
+        iter = fork;
+    }
+
+    if iter.peek_n(0).is_some() {
+        res.push(tt::Subtree { delimiter: None, token_trees: iter.into_iter().cloned().collect() });
+    }
+
+    res
+}
+
 impl TokenMap {
     pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
         let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {
             TokenTextRange::Token(it) => *it == relative_range,
-            TokenTextRange::Delimiter(open, close) => {
-                *open == relative_range || *close == relative_range
+            TokenTextRange::Delimiter(it) => {
+                let open = TextRange::at(it.start(), 1.into());
+                let close = TextRange::at(it.end() - TextSize::of('}'), 1.into());
+                open == relative_range || close == relative_range
             }
         })?;
         Some(token_id)
@@ -137,15 +178,17 @@ impl TokenMap {
         close_relative_range: TextRange,
     ) -> usize {
         let res = self.entries.len();
-        self.entries
-            .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range)));
+        let cover = open_relative_range.cover(close_relative_range);
+
+        self.entries.push((token_id, TokenTextRange::Delimiter(cover)));
         res
     }
 
     fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) {
         let (_, token_text_range) = &mut self.entries[idx];
-        if let TokenTextRange::Delimiter(dim, _) = token_text_range {
-            *token_text_range = TokenTextRange::Delimiter(*dim, close_relative_range);
+        if let TokenTextRange::Delimiter(dim) = token_text_range {
+            let cover = dim.cover(close_relative_range);
+            *token_text_range = TokenTextRange::Delimiter(cover);
         }
     }
 
@@ -170,7 +213,7 @@ fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
 
     // Quote the string
     // Note that `tt::Literal` expect an escaped string
-    let text = format!("{:?}", text.escape_default().to_string());
+    let text = format!("{:?}", text.escape_debug().to_string());
     text.into()
 }
 
@@ -501,7 +544,7 @@ impl SrcToken for SynToken {
         }
     }
     fn to_text(&self) -> SmolStr {
-        self.token().text().clone()
+        self.token().text().into()
     }
 }
 
@@ -676,10 +719,8 @@ impl<'a> TreeSink for TtTreeSink<'a> {
             self.text_pos += TextSize::of(text);
         }
 
-        let text = SmolStr::new(self.buf.as_str());
+        self.inner.token(kind, self.buf.as_str());
         self.buf.clear();
-        self.inner.token(kind, text);
-
         // Add whitespace between adjoint puncts
         let next = last.bump();
         if let (
diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs
index bd2977ebd..5c641ebf2 100644
--- a/crates/mbe/src/tests.rs
+++ b/crates/mbe/src/tests.rs
@@ -6,7 +6,7 @@ use syntax::{
     SyntaxKind::{ERROR, IDENT},
     SyntaxNode, WalkEvent, T,
 };
-use test_utils::assert_eq_text;
+use test_utils::{assert_eq_text, mark};
 
 use super::*;
 
@@ -33,19 +33,18 @@ mod rule_parsing {
 
     #[test]
     fn test_invalid_arms() {
-        fn check(macro_body: &str, err: &str) {
+        fn check(macro_body: &str, err: ParseError) {
             let m = parse_macro_arm(macro_body);
-            assert_eq!(m, Err(ParseError::Expected(String::from(err))));
+            assert_eq!(m, Err(err.into()));
         }
+        check("invalid", ParseError::Expected("expected subtree".into()));
 
-        check("invalid", "expected subtree");
+        check("$i:ident => ()", ParseError::Expected("expected subtree".into()));
+        check("($i:ident) ()", ParseError::Expected("expected `=`".into()));
+        check("($($i:ident)_) => ()", ParseError::InvalidRepeat);
 
-        check("$i:ident => ()", "expected subtree");
-        check("($i:ident) ()", "expected `=`");
-        check("($($i:ident)_) => ()", "invalid repeat");
-
-        check("($i) => ($i)", "invalid macro definition");
-        check("($i:) => ($i)", "invalid macro definition");
+        check("($i) => ($i)", ParseError::UnexpectedToken("bad fragment specifier 1".into()));
+        check("($i:) => ($i)", ParseError::UnexpectedToken("bad fragment specifier 1".into()));
     }
 
     fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> {
@@ -458,6 +457,17 @@ fn test_match_group_with_multichar_sep() {
 }
 
 #[test]
+fn test_match_group_with_multichar_sep2() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            (fn $name:ident {$($i:literal)&&*} ) => ( fn $name() -> bool { $($i)&&*} );
+        }"#,
+    )
+    .assert_expand_items("foo! (fn baz {true && true} );", "fn baz () -> bool {true &&true}");
+}
+
+#[test]
 fn test_match_group_zero_match() {
     parse_macro(
         r#"
@@ -675,6 +685,36 @@ fn test_match_literal() {
     .assert_expand_items("foo! ['('];", "fn foo () {}");
 }
 
+#[test]
+fn test_parse_macro_def_simple() {
+    mark::check!(parse_macro_def_simple);
+
+    parse_macro2(
+        r#"
+macro foo($id:ident) {
+    fn $id() {}
+}
+"#,
+    )
+    .assert_expand_items("foo!(bar);", "fn bar () {}");
+}
+
+#[test]
+fn test_parse_macro_def_rules() {
+    mark::check!(parse_macro_def_rules);
+
+    parse_macro2(
+        r#"
+macro foo {
+    ($id:ident) => {
+        fn $id() {}
+    }
+}
+"#,
+    )
+    .assert_expand_items("foo!(bar);", "fn bar () {}");
+}
+
 // The following tests are port from intellij-rust directly
 // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
 
@@ -941,6 +981,29 @@ fn test_meta_doc_comments() {
 }
 
 #[test]
+fn test_meta_doc_comments_non_latin() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($(#[$ i:meta])+) => (
+                $(#[$ i])+
+                fn bar() {}
+            )
+        }
+"#,
+    ).
+    assert_expand_items(
+        r#"foo! {
+            /// 錦瑟無端五十弦，一弦一柱思華年。
+            /**
+                莊生曉夢迷蝴蝶，望帝春心託杜鵑。
+            */
+        }"#,
+        "# [doc = \" 錦瑟無端五十弦，一弦一柱思華年。\"] # [doc = \"\\\\n                莊生曉夢迷蝴蝶，望帝春心託杜鵑。\\\\n            \"] fn bar () {}",
+    );
+}
+
+#[test]
 fn test_tt_block() {
     parse_macro(
         r#"
@@ -1215,6 +1278,18 @@ macro_rules! m {
     .is_some());
 }
 
+#[test]
+fn test_match_is_not_greedy() {
+    parse_macro(
+        r#"
+macro_rules! foo {
+    ($($i:ident $(,)*),*) => {};
+}
+"#,
+    )
+    .assert_expand_items(r#"foo!(a,b);"#, r#""#);
+}
+
 // The following tests are based on real world situations
 #[test]
 fn test_vec() {
@@ -1699,95 +1774,122 @@ pub(crate) struct MacroFixture {
     rules: MacroRules,
 }
 
-impl MacroFixture {
-    pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree {
-        self.try_expand_tt(invocation).unwrap()
-    }
-
-    fn try_expand_tt(&self, invocation: &str) -> Result<tt::Subtree, ExpandError> {
-        let source_file = ast::SourceFile::parse(invocation).tree();
-        let macro_invocation =
-            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
-
-        let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
-            .ok_or_else(|| ExpandError::ConversionError)?;
+pub(crate) struct MacroFixture2 {
+    rules: MacroDef,
+}
 
-        self.rules.expand(&invocation_tt).result()
-    }
+macro_rules! impl_fixture {
+    ($name:ident) => {
+        impl $name {
+            pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree {
+                self.try_expand_tt(invocation).unwrap()
+            }
 
-    fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {
-        assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err));
-    }
+            fn try_expand_tt(&self, invocation: &str) -> Result<tt::Subtree, ExpandError> {
+                let source_file = ast::SourceFile::parse(invocation).tree();
+                let macro_invocation =
+                    source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
-    fn expand_items(&self, invocation: &str) -> SyntaxNode {
-        let expanded = self.expand_tt(invocation);
-        token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node()
-    }
+                let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
+                    .ok_or_else(|| ExpandError::ConversionError)?;
 
-    fn expand_statements(&self, invocation: &str) -> SyntaxNode {
-        let expanded = self.expand_tt(invocation);
-        token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node()
-    }
+                self.rules.expand(&invocation_tt).result()
+            }
 
-    fn expand_expr(&self, invocation: &str) -> SyntaxNode {
-        let expanded = self.expand_tt(invocation);
-        token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node()
-    }
+            #[allow(unused)]
+            fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {
+                assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err));
+            }
 
-    fn assert_expand_tt(&self, invocation: &str, expected: &str) {
-        let expansion = self.expand_tt(invocation);
-        assert_eq!(expansion.to_string(), expected);
-    }
+            #[allow(unused)]
+            fn expand_items(&self, invocation: &str) -> SyntaxNode {
+                let expanded = self.expand_tt(invocation);
+                token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node()
+            }
 
-    fn assert_expand(&self, invocation: &str, expected: &str) {
-        let expansion = self.expand_tt(invocation);
-        let actual = format!("{:?}", expansion);
-        test_utils::assert_eq_text!(&expected.trim(), &actual.trim());
-    }
+            #[allow(unused)]
+            fn expand_statements(&self, invocation: &str) -> SyntaxNode {
+                let expanded = self.expand_tt(invocation);
+                token_tree_to_syntax_node(&expanded, FragmentKind::Statements)
+                    .unwrap()
+                    .0
+                    .syntax_node()
+            }
 
-    fn assert_expand_items(&self, invocation: &str, expected: &str) -> &MacroFixture {
-        self.assert_expansion(FragmentKind::Items, invocation, expected);
-        self
-    }
+            #[allow(unused)]
+            fn expand_expr(&self, invocation: &str) -> SyntaxNode {
+                let expanded = self.expand_tt(invocation);
+                token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node()
+            }
 
-    fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &MacroFixture {
-        self.assert_expansion(FragmentKind::Statements, invocation, expected);
-        self
-    }
+            #[allow(unused)]
+            fn assert_expand_tt(&self, invocation: &str, expected: &str) {
+                let expansion = self.expand_tt(invocation);
+                assert_eq!(expansion.to_string(), expected);
+            }
 
-    fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) {
-        let expanded = self.expand_tt(invocation);
-        assert_eq!(expanded.to_string(), expected);
-
-        let expected = expected.replace("$crate", "C_C__C");
-
-        // wrap the given text to a macro call
-        let expected = {
-            let wrapped = format!("wrap_macro!( {} )", expected);
-            let wrapped = ast::SourceFile::parse(&wrapped);
-            let wrapped =
-                wrapped.tree().syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
-            let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0;
-            wrapped.delimiter = None;
-            wrapped
-        };
+            #[allow(unused)]
+            fn assert_expand(&self, invocation: &str, expected: &str) {
+                let expansion = self.expand_tt(invocation);
+                let actual = format!("{:?}", expansion);
+                test_utils::assert_eq_text!(&expected.trim(), &actual.trim());
+            }
 
-        let expanded_tree = token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node();
-        let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string();
+            fn assert_expand_items(&self, invocation: &str, expected: &str) -> &$name {
+                self.assert_expansion(FragmentKind::Items, invocation, expected);
+                self
+            }
 
-        let expected_tree = token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node();
-        let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string();
+            #[allow(unused)]
+            fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &$name {
+                self.assert_expansion(FragmentKind::Statements, invocation, expected);
+                self
+            }
 
-        let expected_tree = expected_tree.replace("C_C__C", "$crate");
-        assert_eq!(
-            expanded_tree, expected_tree,
-            "\nleft:\n{}\nright:\n{}",
-            expanded_tree, expected_tree,
-        );
-    }
+            fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) {
+                let expanded = self.expand_tt(invocation);
+                assert_eq!(expanded.to_string(), expected);
+
+                let expected = expected.replace("$crate", "C_C__C");
+
+                // wrap the given text to a macro call
+                let expected = {
+                    let wrapped = format!("wrap_macro!( {} )", expected);
+                    let wrapped = ast::SourceFile::parse(&wrapped);
+                    let wrapped = wrapped
+                        .tree()
+                        .syntax()
+                        .descendants()
+                        .find_map(ast::TokenTree::cast)
+                        .unwrap();
+                    let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0;
+                    wrapped.delimiter = None;
+                    wrapped
+                };
+
+                let expanded_tree =
+                    token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node();
+                let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string();
+
+                let expected_tree =
+                    token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node();
+                let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string();
+
+                let expected_tree = expected_tree.replace("C_C__C", "$crate");
+                assert_eq!(
+                    expanded_tree, expected_tree,
+                    "\nleft:\n{}\nright:\n{}",
+                    expanded_tree, expected_tree,
+                );
+            }
+        }
+    };
 }
 
-fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree {
+impl_fixture!(MacroFixture);
+impl_fixture!(MacroFixture2);
+
+fn parse_macro_rules_to_tt(ra_fixture: &str) -> tt::Subtree {
     let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
     let macro_definition =
         source_file.syntax().descendants().find_map(ast::MacroRules::cast).unwrap();
@@ -1804,14 +1906,36 @@ fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree {
     definition_tt
 }
 
+fn parse_macro_def_to_tt(ra_fixture: &str) -> tt::Subtree {
+    let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
+    let macro_definition =
+        source_file.syntax().descendants().find_map(ast::MacroDef::cast).unwrap();
+
+    let (definition_tt, _) = ast_to_token_tree(&macro_definition.body().unwrap()).unwrap();
+
+    let parsed =
+        parse_to_token_tree(&ra_fixture[macro_definition.body().unwrap().syntax().text_range()])
+            .unwrap()
+            .0;
+    assert_eq!(definition_tt, parsed);
+
+    definition_tt
+}
+
 pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture {
-    let definition_tt = parse_macro_to_tt(ra_fixture);
+    let definition_tt = parse_macro_rules_to_tt(ra_fixture);
     let rules = MacroRules::parse(&definition_tt).unwrap();
     MacroFixture { rules }
 }
 
+pub(crate) fn parse_macro2(ra_fixture: &str) -> MacroFixture2 {
+    let definition_tt = parse_macro_def_to_tt(ra_fixture);
+    let rules = MacroDef::parse(&definition_tt).unwrap();
+    MacroFixture2 { rules }
+}
+
 pub(crate) fn parse_macro_error(ra_fixture: &str) -> ParseError {
-    let definition_tt = parse_macro_to_tt(ra_fixture);
+    let definition_tt = parse_macro_rules_to_tt(ra_fixture);
 
     match MacroRules::parse(&definition_tt) {
         Ok(_) => panic!("Expect error"),
diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs
index 46c420718..a362d31fc 100644
--- a/crates/mbe/src/tt_iter.rs
+++ b/crates/mbe/src/tt_iter.rs
@@ -1,5 +1,20 @@
 //! FIXME: write short doc here
 
+use crate::{subtree_source::SubtreeTokenSource, ExpandError, ExpandResult};
+
+use parser::TreeSink;
+use syntax::SyntaxKind;
+use tt::buffer::{Cursor, TokenBuffer};
+
+macro_rules! err {
+    () => {
+        ExpandError::BindingError(format!(""))
+    };
+    ($($tt:tt)*) => {
+        ExpandError::BindingError(format!($($tt)*))
+    };
+}
+
 #[derive(Debug, Clone)]
 pub(crate) struct TtIter<'a> {
     pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
@@ -56,6 +71,68 @@ impl<'a> TtIter<'a> {
         }
     }
 
+    pub(crate) fn expect_fragment(
+        &mut self,
+        fragment_kind: parser::FragmentKind,
+    ) -> ExpandResult<Option<tt::TokenTree>> {
+        struct OffsetTokenSink<'a> {
+            cursor: Cursor<'a>,
+            error: bool,
+        }
+
+        impl<'a> TreeSink for OffsetTokenSink<'a> {
+            fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+                if kind == SyntaxKind::LIFETIME_IDENT {
+                    n_tokens = 2;
+                }
+                for _ in 0..n_tokens {
+                    self.cursor = self.cursor.bump_subtree();
+                }
+            }
+            fn start_node(&mut self, _kind: SyntaxKind) {}
+            fn finish_node(&mut self) {}
+            fn error(&mut self, _error: parser::ParseError) {
+                self.error = true;
+            }
+        }
+
+        let buffer = TokenBuffer::from_tokens(&self.inner.as_slice());
+        let mut src = SubtreeTokenSource::new(&buffer);
+        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
+
+        parser::parse_fragment(&mut src, &mut sink, fragment_kind);
+
+        let mut err = None;
+        if !sink.cursor.is_root() || sink.error {
+            err = Some(err!("expected {:?}", fragment_kind));
+        }
+
+        let mut curr = buffer.begin();
+        let mut res = vec![];
+
+        if sink.cursor.is_root() {
+            while curr != sink.cursor {
+                if let Some(token) = curr.token_tree() {
+                    res.push(token);
+                }
+                curr = curr.bump();
+            }
+        }
+        self.inner = self.inner.as_slice()[res.len()..].iter();
+        if res.len() == 0 && err.is_none() {
+            err = Some(err!("no tokens consumed"));
+        }
+        let res = match res.len() {
+            1 => Some(res[0].cloned()),
+            0 => None,
+            _ => Some(tt::TokenTree::Subtree(tt::Subtree {
+                delimiter: None,
+                token_trees: res.into_iter().map(|it| it.cloned()).collect(),
+            })),
+        };
+        ExpandResult { value: res, err }
+    }
+
     pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> {
         self.inner.as_slice().get(n)
     }