Merge remote-tracking branch 'upstream/master' into 503-hover-doc-links

10 files changed, 4396 insertions, 0 deletions

diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml
new file mode 100644
index 000000000..1aba8b7c4
--- /dev/null
+++ b/crates/mbe/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "mbe"
+version = "0.0.0"
+license = "MIT OR Apache-2.0"
+authors = ["rust-analyzer developers"]
+edition = "2018"
+
+[lib]
+doctest = false
+
+[dependencies]
+rustc-hash = "1.1.0"
+smallvec = "1.2.0"
+log = "0.4.8"
+
+syntax = { path = "../syntax" }
+parser = { path = "../parser" }
+tt = { path = "../tt" }
+
+[dev-dependencies]
+test_utils = { path = "../test_utils" }
diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
new file mode 100644
index 000000000..f854ca09a
--- /dev/null
+++ b/crates/mbe/src/lib.rs
@@ -0,0 +1,278 @@
+//! `mbe` (short for Macro By Example) crate contains code for handling
+//! `macro_rules` macros. It uses `TokenTree` (from `tt` package) as the
+//! interface, although it contains some code to bridge `SyntaxNode`s and
+//! `TokenTree`s as well!
+
+mod parser;
+mod mbe_expander;
+mod syntax_bridge;
+mod tt_iter;
+mod subtree_source;
+
+#[cfg(test)]
+mod tests;
+
+pub use tt::{Delimiter, Punct};
+
+use crate::{
+    parser::{parse_pattern, Op},
+    tt_iter::TtIter,
+};
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ParseError {
+    Expected(String),
+    RepetitionEmtpyTokenTree,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum ExpandError {
+    NoMatchingRule,
+    UnexpectedToken,
+    BindingError(String),
+    ConversionError,
+    InvalidRepeat,
+    ProcMacroError(tt::ExpansionError),
+}
+
+impl From<tt::ExpansionError> for ExpandError {
+    fn from(it: tt::ExpansionError) -> Self {
+        ExpandError::ProcMacroError(it)
+    }
+}
+
+pub use crate::syntax_bridge::{
+    ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node,
+    TokenMap,
+};
+
+/// This struct contains AST for a single `macro_rules` definition. What might
+/// be very confusing is that AST has almost exactly the same shape as
+/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
+/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MacroRules {
+    rules: Vec<Rule>,
+    /// Highest id of the token we have in TokenMap
+    shift: Shift,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct Rule {
+    lhs: tt::Subtree,
+    rhs: tt::Subtree,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct Shift(u32);
+
+impl Shift {
+    fn new(tt: &tt::Subtree) -> Shift {
+        // Note that TokenId is started from zero,
+        // We have to add 1 to prevent duplication.
+        let value = max_id(tt).map_or(0, |it| it + 1);
+        return Shift(value);
+
+        // Find the max token id inside a subtree
+        fn max_id(subtree: &tt::Subtree) -> Option<u32> {
+            subtree
+                .token_trees
+                .iter()
+                .filter_map(|tt| match tt {
+                    tt::TokenTree::Subtree(subtree) => {
+                        let tree_id = max_id(subtree);
+                        match subtree.delimiter {
+                            Some(it) if it.id != tt::TokenId::unspecified() => {
+                                Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0)))
+                            }
+                            _ => tree_id,
+                        }
+                    }
+                    tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
+                        if ident.id != tt::TokenId::unspecified() =>
+                    {
+                        Some(ident.id.0)
+                    }
+                    _ => None,
+                })
+                .max()
+        }
+    }
+
+    /// Shift given TokenTree token id
+    fn shift_all(self, tt: &mut tt::Subtree) {
+        for t in tt.token_trees.iter_mut() {
+            match t {
+                tt::TokenTree::Leaf(leaf) => match leaf {
+                    tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id),
+                    tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id),
+                    tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id),
+                },
+                tt::TokenTree::Subtree(tt) => {
+                    if let Some(it) = tt.delimiter.as_mut() {
+                        it.id = self.shift(it.id);
+                    };
+                    self.shift_all(tt)
+                }
+            }
+        }
+    }
+
+    fn shift(self, id: tt::TokenId) -> tt::TokenId {
+        if id == tt::TokenId::unspecified() {
+            return id;
+        }
+        tt::TokenId(id.0 + self.0)
+    }
+
+    fn unshift(self, id: tt::TokenId) -> Option<tt::TokenId> {
+        id.0.checked_sub(self.0).map(tt::TokenId)
+    }
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum Origin {
+    Def,
+    Call,
+}
+
+impl MacroRules {
+    pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
+        // Note: this parsing can be implemented using mbe machinery itself, by
+        // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
+        // manually seems easier.
+        let mut src = TtIter::new(tt);
+        let mut rules = Vec::new();
+        while src.len() > 0 {
+            let rule = Rule::parse(&mut src)?;
+            rules.push(rule);
+            if let Err(()) = src.expect_char(';') {
+                if src.len() > 0 {
+                    return Err(ParseError::Expected("expected `:`".to_string()));
+                }
+                break;
+            }
+        }
+
+        for rule in rules.iter() {
+            validate(&rule.lhs)?;
+        }
+
+        Ok(MacroRules { rules, shift: Shift::new(tt) })
+    }
+
+    pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+        // apply shift
+        let mut tt = tt.clone();
+        self.shift.shift_all(&mut tt);
+        mbe_expander::expand(self, &tt)
+    }
+
+    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
+        self.shift.shift(id)
+    }
+
+    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) {
+        match self.shift.unshift(id) {
+            Some(id) => (id, Origin::Call),
+            None => (id, Origin::Def),
+        }
+    }
+}
+
+impl Rule {
+    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
+        let mut lhs = src
+            .expect_subtree()
+            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
+            .clone();
+        lhs.delimiter = None;
+        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
+        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
+        let mut rhs = src
+            .expect_subtree()
+            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
+            .clone();
+        rhs.delimiter = None;
+        Ok(crate::Rule { lhs, rhs })
+    }
+}
+
+fn to_parse_error(e: ExpandError) -> ParseError {
+    let msg = match e {
+        ExpandError::InvalidRepeat => "invalid repeat".to_string(),
+        _ => "invalid macro definition".to_string(),
+    };
+    ParseError::Expected(msg)
+}
+
+fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
+    for op in parse_pattern(pattern) {
+        let op = op.map_err(to_parse_error)?;
+
+        match op {
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) => validate(subtree)?,
+            Op::Repeat { subtree, separator, .. } => {
+                // Checks that no repetition which could match an empty token
+                // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558
+
+                if separator.is_none() {
+                    if parse_pattern(subtree).all(|child_op| {
+                        match child_op.map_err(to_parse_error) {
+                            Ok(Op::Var { kind, .. }) => {
+                                // vis is optional
+                                if kind.map_or(false, |it| it == "vis") {
+                                    return true;
+                                }
+                            }
+                            Ok(Op::Repeat { kind, .. }) => {
+                                return matches!(
+                                    kind,
+                                    parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne
+                                )
+                            }
+                            _ => {}
+                        }
+                        false
+                    }) {
+                        return Err(ParseError::RepetitionEmtpyTokenTree);
+                    }
+                }
+                validate(subtree)?
+            }
+            _ => (),
+        }
+    }
+    Ok(())
+}
+
+#[derive(Debug)]
+pub struct ExpandResult<T>(pub T, pub Option<ExpandError>);
+
+impl<T> ExpandResult<T> {
+    pub fn ok(t: T) -> ExpandResult<T> {
+        ExpandResult(t, None)
+    }
+
+    pub fn only_err(err: ExpandError) -> ExpandResult<T>
+    where
+        T: Default,
+    {
+        ExpandResult(Default::default(), Some(err))
+    }
+
+    pub fn map<U>(self, f: impl FnOnce(T) -> U) -> ExpandResult<U> {
+        ExpandResult(f(self.0), self.1)
+    }
+
+    pub fn result(self) -> Result<T, ExpandError> {
+        self.1.map(Err).unwrap_or(Ok(self.0))
+    }
+}
+
+impl<T: Default> From<Result<T, ExpandError>> for ExpandResult<T> {
+    fn from(result: Result<T, ExpandError>) -> ExpandResult<T> {
+        result
+            .map_or_else(|e| ExpandResult(Default::default(), Some(e)), |it| ExpandResult(it, None))
+    }
+}
diff --git a/crates/mbe/src/mbe_expander.rs b/crates/mbe/src/mbe_expander.rs
new file mode 100644
index 000000000..1ad8b9f8a
--- /dev/null
+++ b/crates/mbe/src/mbe_expander.rs
@@ -0,0 +1,180 @@
+//! This module takes a (parsed) definition of `macro_rules` invocation, a
+//! `tt::TokenTree` representing an argument of macro invocation, and produces a
+//! `tt::TokenTree` for the result of the expansion.
+
+mod matcher;
+mod transcriber;
+
+use rustc_hash::FxHashMap;
+use syntax::SmolStr;
+
+use crate::{ExpandError, ExpandResult};
+
+pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+    expand_rules(&rules.rules, input)
+}
+
+fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+    let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
+    for rule in rules {
+        let new_match = match matcher::match_(&rule.lhs, input) {
+            Ok(m) => m,
+            Err(_e) => {
+                // error in pattern parsing
+                continue;
+            }
+        };
+        if new_match.err.is_none() {
+            // If we find a rule that applies without errors, we're done.
+            // Unconditionally returning the transcription here makes the
+            // `test_repeat_bad_var` test fail.
+            let ExpandResult(res, transcribe_err) =
+                transcriber::transcribe(&rule.rhs, &new_match.bindings);
+            if transcribe_err.is_none() {
+                return ExpandResult::ok(res);
+            }
+        }
+        // Use the rule if we matched more tokens, or had fewer errors
+        if let Some((prev_match, _)) = &match_ {
+            if (new_match.unmatched_tts, new_match.err_count)
+                < (prev_match.unmatched_tts, prev_match.err_count)
+            {
+                match_ = Some((new_match, rule));
+            }
+        } else {
+            match_ = Some((new_match, rule));
+        }
+    }
+    if let Some((match_, rule)) = match_ {
+        // if we got here, there was no match without errors
+        let ExpandResult(result, transcribe_err) =
+            transcriber::transcribe(&rule.rhs, &match_.bindings);
+        ExpandResult(result, match_.err.or(transcribe_err))
+    } else {
+        ExpandResult(tt::Subtree::default(), Some(ExpandError::NoMatchingRule))
+    }
+}
+
+/// The actual algorithm for expansion is not too hard, but is pretty tricky.
+/// `Bindings` structure is the key to understanding what we are doing here.
+///
+/// On the high level, it stores mapping from meta variables to the bits of
+/// syntax it should be substituted with. For example, if `$e:expr` is matched
+/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
+///
+/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
+///
+/// ```not_rust
+/// macro_rules! foo {
+///     ($($ i:ident $($ e:expr),*);*) => {
+///         $(fn $ i() { $($ e);*; })*
+///     }
+/// }
+/// foo! { foo 1,2,3; bar 4,5,6 }
+/// ```
+///
+/// Here, the `$i` meta variable is matched first with `foo` and then with
+/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
+///
+/// To represent such "multi-mappings", we use a recursive structures: we map
+/// variables not to values, but to *lists* of values or other lists (that is,
+/// to the trees).
+///
+/// For the above example, the bindings would store
+///
+/// ```not_rust
+/// i -> [foo, bar]
+/// e -> [[1, 2, 3], [4, 5, 6]]
+/// ```
+///
+/// We construct `Bindings` in the `match_lhs`. The interesting case is
+/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
+/// nesting structure.
+///
+/// The other side of the puzzle is `expand_subtree`, where we use the bindings
+/// to substitute meta variables in the output template. When expanding, we
+/// maintain a `nesting` stack of indices which tells us which occurrence from
+/// the `Bindings` we should take. We push to the stack when we enter a
+/// repetition.
+///
+/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
+/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
+/// many is not a plain `usize`, but an `&[usize]`.
+#[derive(Debug, Default)]
+struct Bindings {
+    inner: FxHashMap<SmolStr, Binding>,
+}
+
+#[derive(Debug)]
+enum Binding {
+    Fragment(Fragment),
+    Nested(Vec<Binding>),
+    Empty,
+}
+
+#[derive(Debug, Clone)]
+enum Fragment {
+    /// token fragments are just copy-pasted into the output
+    Tokens(tt::TokenTree),
+    /// Ast fragments are inserted with fake delimiters, so as to make things
+    /// like `$i * 2` where `$i = 1 + 1` work as expectd.
+    Ast(tt::TokenTree),
+}
+
+#[cfg(test)]
+mod tests {
+    use syntax::{ast, AstNode};
+
+    use super::*;
+    use crate::ast_to_token_tree;
+
+    #[test]
+    fn test_expand_rule() {
+        assert_err(
+            "($($i:ident);*) => ($i)",
+            "foo!{a}",
+            ExpandError::BindingError(String::from(
+                "expected simple binding, found nested binding `i`",
+            )),
+        );
+
+        // FIXME:
+        // Add an err test case for ($($i:ident)) => ($())
+    }
+
+    fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) {
+        assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation).1, Some(err));
+    }
+
+    fn format_macro(macro_body: &str) -> String {
+        format!(
+            "
+        macro_rules! foo {{
+            {}
+        }}
+",
+            macro_body
+        )
+    }
+
+    fn create_rules(macro_definition: &str) -> crate::MacroRules {
+        let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap();
+        let macro_definition =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (definition_tt, _) =
+            ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
+        crate::MacroRules::parse(&definition_tt).unwrap()
+    }
+
+    fn expand_first(rules: &crate::MacroRules, invocation: &str) -> ExpandResult<tt::Subtree> {
+        let source_file = ast::SourceFile::parse(invocation).ok().unwrap();
+        let macro_invocation =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (invocation_tt, _) =
+            ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap();
+
+        expand_rules(&rules.rules, &invocation_tt)
+    }
+}
diff --git a/crates/mbe/src/mbe_expander/matcher.rs b/crates/mbe/src/mbe_expander/matcher.rs
new file mode 100644
index 000000000..b698b9832
--- /dev/null
+++ b/crates/mbe/src/mbe_expander/matcher.rs
@@ -0,0 +1,477 @@
+//! FIXME: write short doc here
+
+use crate::{
+    mbe_expander::{Binding, Bindings, Fragment},
+    parser::{parse_pattern, Op, RepeatKind, Separator},
+    subtree_source::SubtreeTokenSource,
+    tt_iter::TtIter,
+    ExpandError,
+};
+
+use super::ExpandResult;
+use parser::{FragmentKind::*, TreeSink};
+use syntax::{SmolStr, SyntaxKind};
+use tt::buffer::{Cursor, TokenBuffer};
+
+impl Bindings {
+    fn push_optional(&mut self, name: &SmolStr) {
+        // FIXME: Do we have a better way to represent an empty token ?
+        // Insert an empty subtree for empty token
+        let tt = tt::Subtree::default().into();
+        self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
+    }
+
+    fn push_empty(&mut self, name: &SmolStr) {
+        self.inner.insert(name.clone(), Binding::Empty);
+    }
+
+    fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
+        for (key, value) in nested.inner {
+            if !self.inner.contains_key(&key) {
+                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
+            }
+            match self.inner.get_mut(&key) {
+                Some(Binding::Nested(it)) => {
+                    // insert empty nested bindings before this one
+                    while it.len() < idx {
+                        it.push(Binding::Nested(vec![]));
+                    }
+                    it.push(value);
+                }
+                _ => {
+                    return Err(ExpandError::BindingError(format!(
+                        "could not find binding `{}`",
+                        key
+                    )));
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+macro_rules! err {
+    () => {
+        ExpandError::BindingError(format!(""))
+    };
+    ($($tt:tt)*) => {
+        ExpandError::BindingError(format!($($tt)*))
+    };
+}
+
+#[derive(Debug, Default)]
+pub(super) struct Match {
+    pub bindings: Bindings,
+    /// We currently just keep the first error and count the rest to compare matches.
+    pub err: Option<ExpandError>,
+    pub err_count: usize,
+    /// How many top-level token trees were left to match.
+    pub unmatched_tts: usize,
+}
+
+impl Match {
+    pub fn add_err(&mut self, err: ExpandError) {
+        let prev_err = self.err.take();
+        self.err = prev_err.or(Some(err));
+        self.err_count += 1;
+    }
+}
+
+// General note: These functions have two channels to return errors, a `Result`
+// return value and the `&mut Match`. The returned Result is for pattern parsing
+// errors; if a branch of the macro definition doesn't parse, it doesn't make
+// sense to try using it. Matching errors are added to the `Match`. It might
+// make sense to make pattern parsing a separate step?
+
+pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Match, ExpandError> {
+    assert!(pattern.delimiter == None);
+
+    let mut res = Match::default();
+    let mut src = TtIter::new(src);
+
+    match_subtree(&mut res, pattern, &mut src)?;
+
+    if src.len() > 0 {
+        res.unmatched_tts += src.len();
+        res.add_err(err!("leftover tokens"));
+    }
+
+    Ok(res)
+}
+
+fn match_subtree(
+    res: &mut Match,
+    pattern: &tt::Subtree,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::TokenTree(tt::TokenTree::Leaf(lhs)) => {
+                let rhs = match src.expect_leaf() {
+                    Ok(l) => l,
+                    Err(()) => {
+                        res.add_err(err!("expected leaf: `{}`", lhs));
+                        continue;
+                    }
+                };
+                match (lhs, rhs) {
+                    (
+                        tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
+                        tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
+                        tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
+                        tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    _ => {
+                        res.add_err(ExpandError::UnexpectedToken);
+                    }
+                }
+            }
+            Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
+                let rhs = match src.expect_subtree() {
+                    Ok(s) => s,
+                    Err(()) => {
+                        res.add_err(err!("expected subtree"));
+                        continue;
+                    }
+                };
+                if lhs.delimiter_kind() != rhs.delimiter_kind() {
+                    res.add_err(err!("mismatched delimiter"));
+                    continue;
+                }
+                let mut src = TtIter::new(rhs);
+                match_subtree(res, lhs, &mut src)?;
+                if src.len() > 0 {
+                    res.add_err(err!("leftover tokens"));
+                }
+            }
+            Op::Var { name, kind } => {
+                let kind = match kind {
+                    Some(k) => k,
+                    None => {
+                        res.add_err(ExpandError::UnexpectedToken);
+                        continue;
+                    }
+                };
+                let ExpandResult(matched, match_err) = match_meta_var(kind.as_str(), src);
+                match matched {
+                    Some(fragment) => {
+                        res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
+                    }
+                    None if match_err.is_none() => res.bindings.push_optional(name),
+                    _ => {}
+                }
+                if let Some(err) = match_err {
+                    res.add_err(err);
+                }
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                match_repeat(res, subtree, kind, separator, src)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+impl<'a> TtIter<'a> {
+    fn eat_separator(&mut self, separator: &Separator) -> bool {
+        let mut fork = self.clone();
+        let ok = match separator {
+            Separator::Ident(lhs) => match fork.expect_ident() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
+            },
+            Separator::Literal(lhs) => match fork.expect_literal() {
+                Ok(rhs) => match rhs {
+                    tt::Leaf::Literal(rhs) => rhs.text == lhs.text,
+                    tt::Leaf::Ident(rhs) => rhs.text == lhs.text,
+                    tt::Leaf::Punct(_) => false,
+                },
+                _ => false,
+            },
+            Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
+                Ok(rhs) => rhs.char == lhs.char,
+                _ => false,
+            }),
+        };
+        if ok {
+            *self = fork;
+        }
+        ok
+    }
+
+    pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+        match self.peek_n(0) {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
+                return self.expect_lifetime();
+            }
+            _ => (),
+        }
+
+        let tt = self.next().ok_or_else(|| ())?.clone();
+        let punct = match tt {
+            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
+                punct
+            }
+            _ => return Ok(tt),
+        };
+
+        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
+            (
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
+            ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)),
+            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None),
+            _ => return Ok(tt),
+        };
+
+        match (punct.char, second, third) {
+            ('.', '.', Some('.'))
+            | ('.', '.', Some('='))
+            | ('<', '<', Some('='))
+            | ('>', '>', Some('=')) => {
+                let tt2 = self.next().unwrap().clone();
+                let tt3 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into())
+            }
+            ('-', '=', None)
+            | ('-', '>', None)
+            | (':', ':', None)
+            | ('!', '=', None)
+            | ('.', '.', None)
+            | ('*', '=', None)
+            | ('/', '=', None)
+            | ('&', '&', None)
+            | ('&', '=', None)
+            | ('%', '=', None)
+            | ('^', '=', None)
+            | ('+', '=', None)
+            | ('<', '<', None)
+            | ('<', '=', None)
+            | ('=', '=', None)
+            | ('=', '>', None)
+            | ('>', '=', None)
+            | ('>', '>', None)
+            | ('|', '=', None)
+            | ('|', '|', None) => {
+                let tt2 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into())
+            }
+            _ => Ok(tt),
+        }
+    }
+
+    pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+        let punct = self.expect_punct()?;
+        if punct.char != '\'' {
+            return Err(());
+        }
+        let ident = self.expect_ident()?;
+
+        Ok(tt::Subtree {
+            delimiter: None,
+            token_trees: vec![
+                tt::Leaf::Punct(*punct).into(),
+                tt::Leaf::Ident(ident.clone()).into(),
+            ],
+        }
+        .into())
+    }
+
+    pub(crate) fn expect_fragment(
+        &mut self,
+        fragment_kind: parser::FragmentKind,
+    ) -> ExpandResult<Option<tt::TokenTree>> {
+        pub(crate) struct OffsetTokenSink<'a> {
+            pub(crate) cursor: Cursor<'a>,
+            pub(crate) error: bool,
+        }
+
+        impl<'a> TreeSink for OffsetTokenSink<'a> {
+            fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+                if kind == SyntaxKind::LIFETIME {
+                    n_tokens = 2;
+                }
+                for _ in 0..n_tokens {
+                    self.cursor = self.cursor.bump_subtree();
+                }
+            }
+            fn start_node(&mut self, _kind: SyntaxKind) {}
+            fn finish_node(&mut self) {}
+            fn error(&mut self, _error: parser::ParseError) {
+                self.error = true;
+            }
+        }
+
+        let buffer = TokenBuffer::new(&self.inner.as_slice());
+        let mut src = SubtreeTokenSource::new(&buffer);
+        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
+
+        parser::parse_fragment(&mut src, &mut sink, fragment_kind);
+
+        let mut err = None;
+        if !sink.cursor.is_root() || sink.error {
+            err = Some(err!("expected {:?}", fragment_kind));
+        }
+
+        let mut curr = buffer.begin();
+        let mut res = vec![];
+
+        if sink.cursor.is_root() {
+            while curr != sink.cursor {
+                if let Some(token) = curr.token_tree() {
+                    res.push(token);
+                }
+                curr = curr.bump();
+            }
+        }
+        self.inner = self.inner.as_slice()[res.len()..].iter();
+        if res.len() == 0 && err.is_none() {
+            err = Some(err!("no tokens consumed"));
+        }
+        let res = match res.len() {
+            1 => Some(res[0].clone()),
+            0 => None,
+            _ => Some(tt::TokenTree::Subtree(tt::Subtree {
+                delimiter: None,
+                token_trees: res.into_iter().cloned().collect(),
+            })),
+        };
+        ExpandResult(res, err)
+    }
+
+    pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
+        let mut fork = self.clone();
+        match fork.expect_fragment(Visibility) {
+            ExpandResult(tt, None) => {
+                *self = fork;
+                tt
+            }
+            ExpandResult(_, Some(_)) => None,
+        }
+    }
+}
+
+pub(super) fn match_repeat(
+    res: &mut Match,
+    pattern: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a propper macro-by-example implementation
+    let mut limit = 65536;
+    let mut counter = 0;
+
+    for i in 0.. {
+        let mut fork = src.clone();
+
+        if let Some(separator) = &separator {
+            if i != 0 && !fork.eat_separator(separator) {
+                break;
+            }
+        }
+
+        let mut nested = Match::default();
+        match_subtree(&mut nested, pattern, &mut fork)?;
+        if nested.err.is_none() {
+            limit -= 1;
+            if limit == 0 {
+                log::warn!(
+                    "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}",
+                    pattern,
+                    src,
+                    kind,
+                    separator
+                );
+                break;
+            }
+            *src = fork;
+
+            if let Err(err) = res.bindings.push_nested(counter, nested.bindings) {
+                res.add_err(err);
+            }
+            counter += 1;
+            if counter == 1 {
+                if let RepeatKind::ZeroOrOne = kind {
+                    break;
+                }
+            }
+        } else {
+            break;
+        }
+    }
+
+    match (kind, counter) {
+        (RepeatKind::OneOrMore, 0) => {
+            res.add_err(ExpandError::UnexpectedToken);
+        }
+        (_, 0) => {
+            // Collect all empty variables in subtrees
+            let mut vars = Vec::new();
+            collect_vars(&mut vars, pattern)?;
+            for var in vars {
+                res.bindings.push_empty(&var)
+            }
+        }
+        _ => (),
+    }
+    Ok(())
+}
+
+fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragment>> {
+    let fragment = match kind {
+        "path" => Path,
+        "expr" => Expr,
+        "ty" => Type,
+        "pat" => Pattern,
+        "stmt" => Statement,
+        "block" => Block,
+        "meta" => MetaItem,
+        "item" => Item,
+        _ => {
+            let tt_result = match kind {
+                "ident" => input
+                    .expect_ident()
+                    .map(|ident| Some(tt::Leaf::from(ident.clone()).into()))
+                    .map_err(|()| err!("expected ident")),
+                "tt" => input.expect_tt().map(Some).map_err(|()| err!()),
+                "lifetime" => input
+                    .expect_lifetime()
+                    .map(|tt| Some(tt))
+                    .map_err(|()| err!("expected lifetime")),
+                "literal" => input
+                    .expect_literal()
+                    .map(|literal| Some(tt::Leaf::from(literal.clone()).into()))
+                    .map_err(|()| err!()),
+                // `vis` is optional
+                "vis" => match input.eat_vis() {
+                    Some(vis) => Ok(Some(vis)),
+                    None => Ok(None),
+                },
+                _ => Err(ExpandError::UnexpectedToken),
+            };
+            return tt_result.map(|it| it.map(Fragment::Tokens)).into();
+        }
+    };
+    let result = input.expect_fragment(fragment);
+    result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) })
+}
+
+fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::Var { name, .. } => buf.push(name.clone()),
+            Op::TokenTree(tt::TokenTree::Leaf(_)) => (),
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?,
+            Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
+        }
+    }
+    Ok(())
+}
diff --git a/crates/mbe/src/mbe_expander/transcriber.rs b/crates/mbe/src/mbe_expander/transcriber.rs
new file mode 100644
index 000000000..c9525c5bf
--- /dev/null
+++ b/crates/mbe/src/mbe_expander/transcriber.rs
@@ -0,0 +1,254 @@
+//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like
+//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
+
+use syntax::SmolStr;
+
+use super::ExpandResult;
+use crate::{
+    mbe_expander::{Binding, Bindings, Fragment},
+    parser::{parse_template, Op, RepeatKind, Separator},
+    ExpandError,
+};
+
+impl Bindings {
+    fn contains(&self, name: &str) -> bool {
+        self.inner.contains_key(name)
+    }
+
+    fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> {
+        let mut b = self.inner.get(name).ok_or_else(|| {
+            ExpandError::BindingError(format!("could not find binding `{}`", name))
+        })?;
+        for nesting_state in nesting.iter_mut() {
+            nesting_state.hit = true;
+            b = match b {
+                Binding::Fragment(_) => break,
+                Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| {
+                    nesting_state.at_end = true;
+                    ExpandError::BindingError(format!("could not find nested binding `{}`", name))
+                })?,
+                Binding::Empty => {
+                    nesting_state.at_end = true;
+                    return Err(ExpandError::BindingError(format!(
+                        "could not find empty binding `{}`",
+                        name
+                    )));
+                }
+            };
+        }
+        match b {
+            Binding::Fragment(it) => Ok(it),
+            Binding::Nested(_) => Err(ExpandError::BindingError(format!(
+                "expected simple binding, found nested binding `{}`",
+                name
+            ))),
+            Binding::Empty => Err(ExpandError::BindingError(format!(
+                "expected simple binding, found empty binding `{}`",
+                name
+            ))),
+        }
+    }
+}
+
+pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult<tt::Subtree> {
+    assert!(template.delimiter == None);
+    let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() };
+    let mut arena: Vec<tt::TokenTree> = Vec::new();
+    expand_subtree(&mut ctx, template, &mut arena)
+}
+
+#[derive(Debug)]
+struct NestingState {
+    idx: usize,
+    /// `hit` is currently necessary to tell `expand_repeat` if it should stop
+    /// because there is no variable in use by the current repetition
+    hit: bool,
+    /// `at_end` is currently necessary to tell `expand_repeat` if it should stop
+    /// because there is no more value avaible for the current repetition
+    at_end: bool,
+}
+
+#[derive(Debug)]
+struct ExpandCtx<'a> {
+    bindings: &'a Bindings,
+    nesting: Vec<NestingState>,
+}
+
+fn expand_subtree(
+    ctx: &mut ExpandCtx,
+    template: &tt::Subtree,
+    arena: &mut Vec<tt::TokenTree>,
+) -> ExpandResult<tt::Subtree> {
+    // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
+    let start_elements = arena.len();
+    let mut err = None;
+    for op in parse_template(template) {
+        let op = match op {
+            Ok(op) => op,
+            Err(e) => {
+                err = Some(e);
+                break;
+            }
+        };
+        match op {
+            Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => arena.push(tt.clone()),
+            Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
+                let ExpandResult(tt, e) = expand_subtree(ctx, tt, arena);
+                err = err.or(e);
+                arena.push(tt.into());
+            }
+            Op::Var { name, kind: _ } => {
+                let ExpandResult(fragment, e) = expand_var(ctx, name);
+                err = err.or(e);
+                push_fragment(arena, fragment);
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                let ExpandResult(fragment, e) = expand_repeat(ctx, subtree, kind, separator, arena);
+                err = err.or(e);
+                push_fragment(arena, fragment)
+            }
+        }
+    }
+    // drain the elements added in this instance of expand_subtree
+    let tts = arena.drain(start_elements..arena.len()).collect();
+    ExpandResult(tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err)
+}
+
+fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult<Fragment> {
+    if v == "crate" {
+        // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
+        let tt =
+            tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
+                .into();
+        ExpandResult::ok(Fragment::Tokens(tt))
+    } else if !ctx.bindings.contains(v) {
+        // Note that it is possible to have a `$var` inside a macro which is not bound.
+        // For example:
+        // ```
+        // macro_rules! foo {
+        //     ($a:ident, $b:ident, $c:tt) => {
+        //         macro_rules! bar {
+        //             ($bi:ident) => {
+        //                 fn $bi() -> u8 {$c}
+        //             }
+        //         }
+        //     }
+        // ```
+        // We just treat it a normal tokens
+        let tt = tt::Subtree {
+            delimiter: None,
+            token_trees: vec![
+                tt::Leaf::from(tt::Punct {
+                    char: '$',
+                    spacing: tt::Spacing::Alone,
+                    id: tt::TokenId::unspecified(),
+                })
+                .into(),
+                tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
+                    .into(),
+            ],
+        }
+        .into();
+        ExpandResult::ok(Fragment::Tokens(tt))
+    } else {
+        ctx.bindings.get(&v, &mut ctx.nesting).map_or_else(
+            |e| ExpandResult(Fragment::Tokens(tt::TokenTree::empty()), Some(e)),
+            |b| ExpandResult::ok(b.clone()),
+        )
+    }
+}
+
+fn expand_repeat(
+    ctx: &mut ExpandCtx,
+    template: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+    arena: &mut Vec<tt::TokenTree>,
+) -> ExpandResult<Fragment> {
+    let mut buf: Vec<tt::TokenTree> = Vec::new();
+    ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false });
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a proper macro-by-example implementation
+    let limit = 65536;
+    let mut has_seps = 0;
+    let mut counter = 0;
+
+    loop {
+        let ExpandResult(mut t, e) = expand_subtree(ctx, template, arena);
+        let nesting_state = ctx.nesting.last_mut().unwrap();
+        if nesting_state.at_end || !nesting_state.hit {
+            break;
+        }
+        nesting_state.idx += 1;
+        nesting_state.hit = false;
+
+        counter += 1;
+        if counter == limit {
+            log::warn!(
+                "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
+                template,
+                ctx
+            );
+            break;
+        }
+
+        if e.is_some() {
+            continue;
+        }
+
+        t.delimiter = None;
+        push_subtree(&mut buf, t);
+
+        if let Some(ref sep) = separator {
+            match sep {
+                Separator::Ident(ident) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(ident.clone()).into());
+                }
+                Separator::Literal(lit) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(lit.clone()).into());
+                }
+
+                Separator::Puncts(puncts) => {
+                    has_seps = puncts.len();
+                    for punct in puncts {
+                        buf.push(tt::Leaf::from(*punct).into());
+                    }
+                }
+            }
+        }
+
+        if RepeatKind::ZeroOrOne == kind {
+            break;
+        }
+    }
+
+    ctx.nesting.pop().unwrap();
+    for _ in 0..has_seps {
+        buf.pop();
+    }
+
+    // Check if it is a single token subtree without any delimiter
+    // e.g {Delimiter:None> ['>'] /Delimiter:None>}
+    let tt = tt::Subtree { delimiter: None, token_trees: buf }.into();
+
+    if RepeatKind::OneOrMore == kind && counter == 0 {
+        return ExpandResult(Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken));
+    }
+    ExpandResult::ok(Fragment::Tokens(tt))
+}
+
+fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
+    match fragment {
+        Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
+        Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt),
+    }
+}
+
+fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
+    match tt.delimiter {
+        None => buf.extend(tt.token_trees),
+        _ => buf.push(tt.into()),
+    }
+}
diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs
new file mode 100644
index 000000000..6b46a1673
--- /dev/null
+++ b/crates/mbe/src/parser.rs
@@ -0,0 +1,184 @@
+//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
+//! trees.
+
+use smallvec::SmallVec;
+use syntax::SmolStr;
+
+use crate::{tt_iter::TtIter, ExpandError};
+
+#[derive(Debug)]
+pub(crate) enum Op<'a> {
+    Var { name: &'a SmolStr, kind: Option<&'a SmolStr> },
+    Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option<Separator> },
+    TokenTree(&'a tt::TokenTree),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) enum RepeatKind {
+    ZeroOrMore,
+    OneOrMore,
+    ZeroOrOne,
+}
+
+#[derive(Clone, Debug, Eq)]
+pub(crate) enum Separator {
+    Literal(tt::Literal),
+    Ident(tt::Ident),
+    Puncts(SmallVec<[tt::Punct; 3]>),
+}
+
+// Note that when we compare a Separator, we just care about its textual value.
+impl PartialEq for Separator {
+    fn eq(&self, other: &Separator) -> bool {
+        use Separator::*;
+
+        match (self, other) {
+            (Ident(ref a), Ident(ref b)) => a.text == b.text,
+            (Literal(ref a), Literal(ref b)) => a.text == b.text,
+            (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
+                let a_iter = a.iter().map(|a| a.char);
+                let b_iter = b.iter().map(|b| b.char);
+                a_iter.eq(b_iter)
+            }
+            _ => false,
+        }
+    }
+}
+
+pub(crate) fn parse_template(
+    template: &tt::Subtree,
+) -> impl Iterator<Item = Result<Op<'_>, ExpandError>> {
+    parse_inner(template, Mode::Template)
+}
+
+pub(crate) fn parse_pattern(
+    pattern: &tt::Subtree,
+) -> impl Iterator<Item = Result<Op<'_>, ExpandError>> {
+    parse_inner(pattern, Mode::Pattern)
+}
+
+#[derive(Clone, Copy)]
+enum Mode {
+    Pattern,
+    Template,
+}
+
+fn parse_inner(src: &tt::Subtree, mode: Mode) -> impl Iterator<Item = Result<Op<'_>, ExpandError>> {
+    let mut src = TtIter::new(src);
+    std::iter::from_fn(move || {
+        let first = src.next()?;
+        Some(next_op(first, &mut src, mode))
+    })
+}
+
+macro_rules! err {
+    ($($tt:tt)*) => {
+        ExpandError::UnexpectedToken
+    };
+}
+
+macro_rules! bail {
+    ($($tt:tt)*) => {
+        return Err(err!($($tt)*))
+    };
+}
+
+fn next_op<'a>(
+    first: &'a tt::TokenTree,
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Op<'a>, ExpandError> {
+    let res = match first {
+        tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
+            // Note that the '$' itself is a valid token inside macro_rules.
+            let second = match src.next() {
+                None => return Ok(Op::TokenTree(first)),
+                Some(it) => it,
+            };
+            match second {
+                tt::TokenTree::Subtree(subtree) => {
+                    let (separator, kind) = parse_repeat(src)?;
+                    Op::Repeat { subtree, separator, kind }
+                }
+                tt::TokenTree::Leaf(leaf) => match leaf {
+                    tt::Leaf::Punct(..) => return Err(ExpandError::UnexpectedToken),
+                    tt::Leaf::Ident(ident) => {
+                        let name = &ident.text;
+                        let kind = eat_fragment_kind(src, mode)?;
+                        Op::Var { name, kind }
+                    }
+                    tt::Leaf::Literal(lit) => {
+                        if is_boolean_literal(lit) {
+                            let name = &lit.text;
+                            let kind = eat_fragment_kind(src, mode)?;
+                            Op::Var { name, kind }
+                        } else {
+                            bail!("bad var 2");
+                        }
+                    }
+                },
+            }
+        }
+        tt => Op::TokenTree(tt),
+    };
+    Ok(res)
+}
+
+fn eat_fragment_kind<'a>(
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Option<&'a SmolStr>, ExpandError> {
+    if let Mode::Pattern = mode {
+        src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?;
+        let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?;
+        return Ok(Some(&ident.text));
+    };
+    Ok(None)
+}
+
+fn is_boolean_literal(lit: &tt::Literal) -> bool {
+    matches!(lit.text.as_str(), "true" | "false")
+}
+
+fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> {
+    let mut separator = Separator::Puncts(SmallVec::new());
+    for tt in src {
+        let tt = match tt {
+            tt::TokenTree::Leaf(leaf) => leaf,
+            tt::TokenTree::Subtree(_) => return Err(ExpandError::InvalidRepeat),
+        };
+        let has_sep = match &separator {
+            Separator::Puncts(puncts) => !puncts.is_empty(),
+            _ => true,
+        };
+        match tt {
+            tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => {
+                return Err(ExpandError::InvalidRepeat)
+            }
+            tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()),
+            tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
+            tt::Leaf::Punct(punct) => {
+                let repeat_kind = match punct.char {
+                    '*' => RepeatKind::ZeroOrMore,
+                    '+' => RepeatKind::OneOrMore,
+                    '?' => RepeatKind::ZeroOrOne,
+                    _ => {
+                        match &mut separator {
+                            Separator::Puncts(puncts) => {
+                                if puncts.len() == 3 {
+                                    return Err(ExpandError::InvalidRepeat);
+                                }
+                                puncts.push(punct.clone())
+                            }
+                            _ => return Err(ExpandError::InvalidRepeat),
+                        }
+                        continue;
+                    }
+                };
+                let separator = if has_sep { Some(separator) } else { None };
+                return Ok((separator, repeat_kind));
+            }
+        }
+    }
+    Err(ExpandError::InvalidRepeat)
+}
diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs
new file mode 100644
index 000000000..41461b315
--- /dev/null
+++ b/crates/mbe/src/subtree_source.rs
@@ -0,0 +1,197 @@
+//! FIXME: write short doc here
+
+use parser::{Token, TokenSource};
+use std::cell::{Cell, Ref, RefCell};
+use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use tt::buffer::{Cursor, TokenBuffer};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+struct TtToken {
+    pub kind: SyntaxKind,
+    pub is_joint_to_next: bool,
+    pub text: SmolStr,
+}
+
+pub(crate) struct SubtreeTokenSource<'a> {
+    cached_cursor: Cell<Cursor<'a>>,
+    cached: RefCell<Vec<Option<TtToken>>>,
+    curr: (Token, usize),
+}
+
+impl<'a> SubtreeTokenSource<'a> {
+    // Helper function used in test
+    #[cfg(test)]
+    pub fn text(&self) -> SmolStr {
+        match *self.get(self.curr.1) {
+            Some(ref tt) => tt.text.clone(),
+            _ => SmolStr::new(""),
+        }
+    }
+}
+
+impl<'a> SubtreeTokenSource<'a> {
+    pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
+        let cursor = buffer.begin();
+
+        let mut res = SubtreeTokenSource {
+            curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
+            cached_cursor: Cell::new(cursor),
+            cached: RefCell::new(Vec::with_capacity(10)),
+        };
+        res.curr = (res.mk_token(0), 0);
+        res
+    }
+
+    fn mk_token(&self, pos: usize) -> Token {
+        match *self.get(pos) {
+            Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
+            None => Token { kind: EOF, is_jointed_to_next: false },
+        }
+    }
+
+    fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
+        fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
+            let tkn = c.token_tree();
+
+            if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn {
+                if punct.char == '\'' {
+                    let next = c.bump();
+                    if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() {
+                        let res_cursor = next.bump();
+                        let text = SmolStr::new("'".to_string() + &ident.to_string());
+
+                        return Some((res_cursor, text));
+                    } else {
+                        panic!("Next token must be ident : {:#?}", next.token_tree());
+                    }
+                }
+            }
+
+            None
+        }
+
+        if pos < self.cached.borrow().len() {
+            return Ref::map(self.cached.borrow(), |c| &c[pos]);
+        }
+
+        {
+            let mut cached = self.cached.borrow_mut();
+            while pos >= cached.len() {
+                let cursor = self.cached_cursor.get();
+                if cursor.eof() {
+                    cached.push(None);
+                    continue;
+                }
+
+                if let Some((curr, text)) = is_lifetime(cursor) {
+                    cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text }));
+                    self.cached_cursor.set(curr);
+                    continue;
+                }
+
+                match cursor.token_tree() {
+                    Some(tt::TokenTree::Leaf(leaf)) => {
+                        cached.push(Some(convert_leaf(&leaf)));
+                        self.cached_cursor.set(cursor.bump());
+                    }
+                    Some(tt::TokenTree::Subtree(subtree)) => {
+                        self.cached_cursor.set(cursor.subtree().unwrap());
+                        cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
+                    }
+                    None => {
+                        if let Some(subtree) = cursor.end() {
+                            cached.push(Some(convert_delim(subtree.delimiter_kind(), true)));
+                            self.cached_cursor.set(cursor.bump());
+                        }
+                    }
+                }
+            }
+        }
+
+        Ref::map(self.cached.borrow(), |c| &c[pos])
+    }
+}
+
+impl<'a> TokenSource for SubtreeTokenSource<'a> {
+    fn current(&self) -> Token {
+        self.curr.0
+    }
+
+    /// Lookahead n token
+    fn lookahead_nth(&self, n: usize) -> Token {
+        self.mk_token(self.curr.1 + n)
+    }
+
+    /// bump cursor to next token
+    fn bump(&mut self) {
+        if self.current().kind == EOF {
+            return;
+        }
+
+        self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1);
+    }
+
+    /// Is the current token a specified keyword?
+    fn is_keyword(&self, kw: &str) -> bool {
+        match *self.get(self.curr.1) {
+            Some(ref t) => t.text == *kw,
+            _ => false,
+        }
+    }
+}
+
+fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
+    let (kinds, texts) = match d {
+        Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"),
+        Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"),
+        Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"),
+        None => ([L_DOLLAR, R_DOLLAR], ""),
+    };
+
+    let idx = closing as usize;
+    let kind = kinds[idx];
+    let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" };
+    TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
+}
+
+fn convert_literal(l: &tt::Literal) -> TtToken {
+    let kind = lex_single_syntax_kind(&l.text)
+        .map(|(kind, _error)| kind)
+        .filter(|kind| kind.is_literal())
+        .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
+
+    TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
+}
+
+fn convert_ident(ident: &tt::Ident) -> TtToken {
+    let kind = match ident.text.as_ref() {
+        "true" => T![true],
+        "false" => T![false],
+        i if i.starts_with('\'') => LIFETIME,
+        _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
+    };
+
+    TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
+}
+
+fn convert_punct(p: tt::Punct) -> TtToken {
+    let kind = match SyntaxKind::from_char(p.char) {
+        None => panic!("{:#?} is not a valid punct", p),
+        Some(kind) => kind,
+    };
+
+    let text = {
+        let mut buf = [0u8; 4];
+        let s: &str = p.char.encode_utf8(&mut buf);
+        SmolStr::new(s)
+    };
+    TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
+}
+
+fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
+    match leaf {
+        tt::Leaf::Literal(l) => convert_literal(l),
+        tt::Leaf::Ident(ident) => convert_ident(ident),
+        tt::Leaf::Punct(punct) => convert_punct(*punct),
+    }
+}
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
new file mode 100644
index 000000000..a8ad917fb
--- /dev/null
+++ b/crates/mbe/src/syntax_bridge.rs
@@ -0,0 +1,832 @@
+//! FIXME: write short doc here
+
+use parser::{FragmentKind, ParseError, TreeSink};
+use rustc_hash::FxHashMap;
+use syntax::{
+    ast::{self, make::tokens::doc_comment},
+    tokenize, AstToken, Parse, SmolStr, SyntaxKind,
+    SyntaxKind::*,
+    SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, Token as RawToken, T,
+};
+use tt::buffer::{Cursor, TokenBuffer};
+
+use crate::subtree_source::SubtreeTokenSource;
+use crate::ExpandError;
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum TokenTextRange {
+    Token(TextRange),
+    Delimiter(TextRange, TextRange),
+}
+
+impl TokenTextRange {
+    pub fn by_kind(self, kind: SyntaxKind) -> Option<TextRange> {
+        match self {
+            TokenTextRange::Token(it) => Some(it),
+            TokenTextRange::Delimiter(open, close) => match kind {
+                T!['{'] | T!['('] | T!['['] => Some(open),
+                T!['}'] | T![')'] | T![']'] => Some(close),
+                _ => None,
+            },
+        }
+    }
+}
+
+/// Maps `tt::TokenId` to the relative range of the original token.
+#[derive(Debug, PartialEq, Eq, Clone, Default)]
+pub struct TokenMap {
+    /// Maps `tt::TokenId` to the *relative* source range.
+    entries: Vec<(tt::TokenId, TokenTextRange)>,
+}
+
+/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro
+/// will consume).
+pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenMap)> {
+    syntax_node_to_token_tree(ast.syntax())
+}
+
+/// Convert the syntax node to a `TokenTree` (what macro
+/// will consume).
+pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> {
+    let global_offset = node.text_range().start();
+    let mut c = Convertor::new(node, global_offset);
+    let subtree = c.go()?;
+    Some((subtree, c.id_alloc.map))
+}
+
+// The following items are what `rustc` macro can be parsed into :
+// link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141
+// * Expr(P<ast::Expr>)                     -> token_tree_to_expr
+// * Pat(P<ast::Pat>)                       -> token_tree_to_pat
+// * Ty(P<ast::Ty>)                         -> token_tree_to_ty
+// * Stmts(SmallVec<[ast::Stmt; 1]>)        -> token_tree_to_stmts
+// * Items(SmallVec<[P<ast::Item>; 1]>)     -> token_tree_to_items
+//
+// * TraitItems(SmallVec<[ast::TraitItem; 1]>)
+// * AssocItems(SmallVec<[ast::AssocItem; 1]>)
+// * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
+
+pub fn token_tree_to_syntax_node(
+    tt: &tt::Subtree,
+    fragment_kind: FragmentKind,
+) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> {
+    let tmp;
+    let tokens = match tt {
+        tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(),
+        _ => {
+            tmp = [tt.clone().into()];
+            &tmp[..]
+        }
+    };
+    let buffer = TokenBuffer::new(&tokens);
+    let mut token_source = SubtreeTokenSource::new(&buffer);
+    let mut tree_sink = TtTreeSink::new(buffer.begin());
+    parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind);
+    if tree_sink.roots.len() != 1 {
+        return Err(ExpandError::ConversionError);
+    }
+    //FIXME: would be cool to report errors
+    let (parse, range_map) = tree_sink.finish();
+    Ok((parse, range_map))
+}
+
+/// Convert a string to a `TokenTree`
+pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
+    let (tokens, errors) = tokenize(text);
+    if !errors.is_empty() {
+        return None;
+    }
+
+    let mut conv = RawConvertor {
+        text,
+        offset: TextSize::default(),
+        inner: tokens.iter(),
+        id_alloc: TokenIdAlloc {
+            map: Default::default(),
+            global_offset: TextSize::default(),
+            next_id: 0,
+        },
+    };
+
+    let subtree = conv.go()?;
+    Some((subtree, conv.id_alloc.map))
+}
+
+impl TokenMap {
+    pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
+        let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {
+            TokenTextRange::Token(it) => *it == relative_range,
+            TokenTextRange::Delimiter(open, close) => {
+                *open == relative_range || *close == relative_range
+            }
+        })?;
+        Some(token_id)
+    }
+
+    pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TokenTextRange> {
+        let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?;
+        Some(range)
+    }
+
+    fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
+        self.entries.push((token_id, TokenTextRange::Token(relative_range)));
+    }
+
+    fn insert_delim(
+        &mut self,
+        token_id: tt::TokenId,
+        open_relative_range: TextRange,
+        close_relative_range: TextRange,
+    ) -> usize {
+        let res = self.entries.len();
+        self.entries
+            .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range)));
+        res
+    }
+
+    fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) {
+        let (_, token_text_range) = &mut self.entries[idx];
+        if let TokenTextRange::Delimiter(dim, _) = token_text_range {
+            *token_text_range = TokenTextRange::Delimiter(*dim, close_relative_range);
+        }
+    }
+
+    fn remove_delim(&mut self, idx: usize) {
+        // FIXME: This could be accidently quadratic
+        self.entries.remove(idx);
+    }
+}
+
+/// Returns the textual content of a doc comment block as a quoted string
+/// That is, strips leading `///` (or `/**`, etc)
+/// and strips the ending `*/`
+/// And then quote the string, which is needed to convert to `tt::Literal`
+fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
+    let prefix_len = comment.prefix().len();
+    let mut text = &comment.text()[prefix_len..];
+
+    // Remove ending "*/"
+    if comment.kind().shape == ast::CommentShape::Block {
+        text = &text[0..text.len() - 2];
+    }
+
+    // Quote the string
+    // Note that `tt::Literal` expect an escaped string
+    let text = format!("{:?}", text.escape_default().to_string());
+    text.into()
+}
+
+fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>> {
+    let comment = ast::Comment::cast(token.clone())?;
+    let doc = comment.kind().doc?;
+
+    // Make `doc="\" Comments\""
+    let mut meta_tkns = Vec::new();
+    meta_tkns.push(mk_ident("doc"));
+    meta_tkns.push(mk_punct('='));
+    meta_tkns.push(mk_doc_literal(&comment));
+
+    // Make `#![]`
+    let mut token_trees = Vec::new();
+    token_trees.push(mk_punct('#'));
+    if let ast::CommentPlacement::Inner = doc {
+        token_trees.push(mk_punct('!'));
+    }
+    token_trees.push(tt::TokenTree::from(tt::Subtree {
+        delimiter: Some(tt::Delimiter {
+            kind: tt::DelimiterKind::Bracket,
+            id: tt::TokenId::unspecified(),
+        }),
+        token_trees: meta_tkns,
+    }));
+
+    return Some(token_trees);
+
+    // Helper functions
+    fn mk_ident(s: &str) -> tt::TokenTree {
+        tt::TokenTree::from(tt::Leaf::from(tt::Ident {
+            text: s.into(),
+            id: tt::TokenId::unspecified(),
+        }))
+    }
+
+    fn mk_punct(c: char) -> tt::TokenTree {
+        tt::TokenTree::from(tt::Leaf::from(tt::Punct {
+            char: c,
+            spacing: tt::Spacing::Alone,
+            id: tt::TokenId::unspecified(),
+        }))
+    }
+
+    fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
+        let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
+
+        tt::TokenTree::from(tt::Leaf::from(lit))
+    }
+}
+
+struct TokenIdAlloc {
+    map: TokenMap,
+    global_offset: TextSize,
+    next_id: u32,
+}
+
+impl TokenIdAlloc {
+    fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId {
+        let relative_range = absolute_range - self.global_offset;
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+        self.map.insert(token_id, relative_range);
+        token_id
+    }
+
+    fn open_delim(&mut self, open_abs_range: TextRange) -> (tt::TokenId, usize) {
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+        let idx = self.map.insert_delim(
+            token_id,
+            open_abs_range - self.global_offset,
+            open_abs_range - self.global_offset,
+        );
+        (token_id, idx)
+    }
+
+    fn close_delim(&mut self, idx: usize, close_abs_range: Option<TextRange>) {
+        match close_abs_range {
+            None => {
+                self.map.remove_delim(idx);
+            }
+            Some(close) => {
+                self.map.update_close_delim(idx, close - self.global_offset);
+            }
+        }
+    }
+}
+
+/// A Raw Token (straightly from lexer) convertor
+struct RawConvertor<'a> {
+    text: &'a str,
+    offset: TextSize,
+    id_alloc: TokenIdAlloc,
+    inner: std::slice::Iter<'a, RawToken>,
+}
+
+trait SrcToken: std::fmt::Debug {
+    fn kind(&self) -> SyntaxKind;
+
+    fn to_char(&self) -> Option<char>;
+
+    fn to_text(&self) -> SmolStr;
+}
+
+trait TokenConvertor {
+    type Token: SrcToken;
+
+    fn go(&mut self) -> Option<tt::Subtree> {
+        let mut subtree = tt::Subtree::default();
+        subtree.delimiter = None;
+        while self.peek().is_some() {
+            self.collect_leaf(&mut subtree.token_trees);
+        }
+        if subtree.token_trees.is_empty() {
+            return None;
+        }
+        if subtree.token_trees.len() == 1 {
+            if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
+                return Some(first.clone());
+            }
+        }
+        Some(subtree)
+    }
+
+    fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
+        let (token, range) = match self.bump() {
+            None => return,
+            Some(it) => it,
+        };
+
+        let k: SyntaxKind = token.kind();
+        if k == COMMENT {
+            if let Some(tokens) = self.convert_doc_comment(&token) {
+                result.extend(tokens);
+            }
+            return;
+        }
+
+        result.push(if k.is_punct() {
+            assert_eq!(range.len(), TextSize::of('.'));
+            let delim = match k {
+                T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
+                T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
+                T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
+                _ => None,
+            };
+
+            if let Some((kind, closed)) = delim {
+                let mut subtree = tt::Subtree::default();
+                let (id, idx) = self.id_alloc().open_delim(range);
+                subtree.delimiter = Some(tt::Delimiter { kind, id });
+
+                while self.peek().map(|it| it.kind() != closed).unwrap_or(false) {
+                    self.collect_leaf(&mut subtree.token_trees);
+                }
+                let last_range = match self.bump() {
+                    None => {
+                        // For error resilience, we insert an char punct for the opening delim here
+                        self.id_alloc().close_delim(idx, None);
+                        let leaf: tt::Leaf = tt::Punct {
+                            id: self.id_alloc().alloc(range),
+                            char: token.to_char().unwrap(),
+                            spacing: tt::Spacing::Alone,
+                        }
+                        .into();
+                        result.push(leaf.into());
+                        result.extend(subtree.token_trees);
+                        return;
+                    }
+                    Some(it) => it.1,
+                };
+                self.id_alloc().close_delim(idx, Some(last_range));
+                subtree.into()
+            } else {
+                let spacing = match self.peek() {
+                    Some(next)
+                        if next.kind().is_trivia()
+                            || next.kind() == T!['[']
+                            || next.kind() == T!['{']
+                            || next.kind() == T!['('] =>
+                    {
+                        tt::Spacing::Alone
+                    }
+                    Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
+                    _ => tt::Spacing::Alone,
+                };
+                let char = match token.to_char() {
+                    Some(c) => c,
+                    None => {
+                        panic!("Token from lexer must be single char: token = {:#?}", token);
+                    }
+                };
+                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
+            }
+        } else {
+            macro_rules! make_leaf {
+                ($i:ident) => {
+                    tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into()
+                };
+            }
+            let leaf: tt::Leaf = match k {
+                T![true] | T![false] => make_leaf!(Ident),
+                IDENT => make_leaf!(Ident),
+                k if k.is_keyword() => make_leaf!(Ident),
+                k if k.is_literal() => make_leaf!(Literal),
+                LIFETIME => {
+                    let char_unit = TextSize::of('\'');
+                    let r = TextRange::at(range.start(), char_unit);
+                    let apostrophe = tt::Leaf::from(tt::Punct {
+                        char: '\'',
+                        spacing: tt::Spacing::Joint,
+                        id: self.id_alloc().alloc(r),
+                    });
+                    result.push(apostrophe.into());
+
+                    let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
+                    let ident = tt::Leaf::from(tt::Ident {
+                        text: SmolStr::new(&token.to_text()[1..]),
+                        id: self.id_alloc().alloc(r),
+                    });
+                    result.push(ident.into());
+                    return;
+                }
+                _ => return,
+            };
+
+            leaf.into()
+        });
+    }
+
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>;
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
+
+    fn peek(&self) -> Option<Self::Token>;
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc;
+}
+
+impl<'a> SrcToken for (RawToken, &'a str) {
+    fn kind(&self) -> SyntaxKind {
+        self.0.kind
+    }
+
+    fn to_char(&self) -> Option<char> {
+        self.1.chars().next()
+    }
+
+    fn to_text(&self) -> SmolStr {
+        self.1.into()
+    }
+}
+
+impl RawConvertor<'_> {}
+
+impl<'a> TokenConvertor for RawConvertor<'a> {
+    type Token = (RawToken, &'a str);
+
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
+        convert_doc_comment(&doc_comment(token.1))
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
+        let token = self.inner.next()?;
+        let range = TextRange::at(self.offset, token.len);
+        self.offset += token.len;
+
+        Some(((*token, &self.text[range]), range))
+    }
+
+    fn peek(&self) -> Option<Self::Token> {
+        let token = self.inner.as_slice().get(0).cloned();
+
+        token.map(|it| {
+            let range = TextRange::at(self.offset, it.len);
+            (it, &self.text[range])
+        })
+    }
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc {
+        &mut self.id_alloc
+    }
+}
+
+struct Convertor {
+    id_alloc: TokenIdAlloc,
+    current: Option<SyntaxToken>,
+    range: TextRange,
+    punct_offset: Option<(SyntaxToken, TextSize)>,
+}
+
+impl Convertor {
+    fn new(node: &SyntaxNode, global_offset: TextSize) -> Convertor {
+        Convertor {
+            id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
+            current: node.first_token(),
+            range: node.text_range(),
+            punct_offset: None,
+        }
+    }
+}
+
+#[derive(Debug)]
+enum SynToken {
+    Ordiniary(SyntaxToken),
+    Punch(SyntaxToken, TextSize),
+}
+
+impl SynToken {
+    fn token(&self) -> &SyntaxToken {
+        match self {
+            SynToken::Ordiniary(it) => it,
+            SynToken::Punch(it, _) => it,
+        }
+    }
+}
+
+impl SrcToken for SynToken {
+    fn kind(&self) -> SyntaxKind {
+        self.token().kind()
+    }
+    fn to_char(&self) -> Option<char> {
+        match self {
+            SynToken::Ordiniary(_) => None,
+            SynToken::Punch(it, i) => it.text().chars().nth((*i).into()),
+        }
+    }
+    fn to_text(&self) -> SmolStr {
+        self.token().text().clone()
+    }
+}
+
+impl TokenConvertor for Convertor {
+    type Token = SynToken;
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
+        convert_doc_comment(token.token())
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
+        if let Some((punct, offset)) = self.punct_offset.clone() {
+            if usize::from(offset) + 1 < punct.text().len() {
+                let offset = offset + TextSize::of('.');
+                let range = punct.text_range();
+                self.punct_offset = Some((punct.clone(), offset));
+                let range = TextRange::at(range.start() + offset, TextSize::of('.'));
+                return Some((SynToken::Punch(punct, offset), range));
+            }
+        }
+
+        let curr = self.current.clone()?;
+        if !&self.range.contains_range(curr.text_range()) {
+            return None;
+        }
+        self.current = curr.next_token();
+
+        let token = if curr.kind().is_punct() {
+            let range = curr.text_range();
+            let range = TextRange::at(range.start(), TextSize::of('.'));
+            self.punct_offset = Some((curr.clone(), 0.into()));
+            (SynToken::Punch(curr, 0.into()), range)
+        } else {
+            self.punct_offset = None;
+            let range = curr.text_range();
+            (SynToken::Ordiniary(curr), range)
+        };
+
+        Some(token)
+    }
+
+    fn peek(&self) -> Option<Self::Token> {
+        if let Some((punct, mut offset)) = self.punct_offset.clone() {
+            offset = offset + TextSize::of('.');
+            if usize::from(offset) < punct.text().len() {
+                return Some(SynToken::Punch(punct, offset));
+            }
+        }
+
+        let curr = self.current.clone()?;
+        if !self.range.contains_range(curr.text_range()) {
+            return None;
+        }
+
+        let token = if curr.kind().is_punct() {
+            SynToken::Punch(curr, 0.into())
+        } else {
+            SynToken::Ordiniary(curr)
+        };
+        Some(token)
+    }
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc {
+        &mut self.id_alloc
+    }
+}
+
+struct TtTreeSink<'a> {
+    buf: String,
+    cursor: Cursor<'a>,
+    open_delims: FxHashMap<tt::TokenId, TextSize>,
+    text_pos: TextSize,
+    inner: SyntaxTreeBuilder,
+    token_map: TokenMap,
+
+    // Number of roots
+    // Use for detect ill-form tree which is not single root
+    roots: smallvec::SmallVec<[usize; 1]>,
+}
+
+impl<'a> TtTreeSink<'a> {
+    fn new(cursor: Cursor<'a>) -> Self {
+        TtTreeSink {
+            buf: String::new(),
+            cursor,
+            open_delims: FxHashMap::default(),
+            text_pos: 0.into(),
+            inner: SyntaxTreeBuilder::default(),
+            roots: smallvec::SmallVec::new(),
+            token_map: TokenMap::default(),
+        }
+    }
+
+    fn finish(self) -> (Parse<SyntaxNode>, TokenMap) {
+        (self.inner.finish(), self.token_map)
+    }
+}
+
+fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> SmolStr {
+    let texts = match d {
+        Some(tt::DelimiterKind::Parenthesis) => "()",
+        Some(tt::DelimiterKind::Brace) => "{}",
+        Some(tt::DelimiterKind::Bracket) => "[]",
+        None => return "".into(),
+    };
+
+    let idx = closing as usize;
+    let text = &texts[idx..texts.len() - (1 - idx)];
+    text.into()
+}
+
+impl<'a> TreeSink for TtTreeSink<'a> {
+    fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+        if kind == L_DOLLAR || kind == R_DOLLAR {
+            self.cursor = self.cursor.bump_subtree();
+            return;
+        }
+        if kind == LIFETIME {
+            n_tokens = 2;
+        }
+
+        let mut last = self.cursor;
+        for _ in 0..n_tokens {
+            if self.cursor.eof() {
+                break;
+            }
+            last = self.cursor;
+            let text: SmolStr = match self.cursor.token_tree() {
+                Some(tt::TokenTree::Leaf(leaf)) => {
+                    // Mark the range if needed
+                    let (text, id) = match leaf {
+                        tt::Leaf::Ident(ident) => (ident.text.clone(), ident.id),
+                        tt::Leaf::Punct(punct) => {
+                            (SmolStr::new_inline_from_ascii(1, &[punct.char as u8]), punct.id)
+                        }
+                        tt::Leaf::Literal(lit) => (lit.text.clone(), lit.id),
+                    };
+                    let range = TextRange::at(self.text_pos, TextSize::of(text.as_str()));
+                    self.token_map.insert(id, range);
+                    self.cursor = self.cursor.bump();
+                    text
+                }
+                Some(tt::TokenTree::Subtree(subtree)) => {
+                    self.cursor = self.cursor.subtree().unwrap();
+                    if let Some(id) = subtree.delimiter.map(|it| it.id) {
+                        self.open_delims.insert(id, self.text_pos);
+                    }
+                    delim_to_str(subtree.delimiter_kind(), false)
+                }
+                None => {
+                    if let Some(parent) = self.cursor.end() {
+                        self.cursor = self.cursor.bump();
+                        if let Some(id) = parent.delimiter.map(|it| it.id) {
+                            if let Some(open_delim) = self.open_delims.get(&id) {
+                                let open_range = TextRange::at(*open_delim, TextSize::of('('));
+                                let close_range = TextRange::at(self.text_pos, TextSize::of('('));
+                                self.token_map.insert_delim(id, open_range, close_range);
+                            }
+                        }
+                        delim_to_str(parent.delimiter_kind(), true)
+                    } else {
+                        continue;
+                    }
+                }
+            };
+            self.buf += &text;
+            self.text_pos += TextSize::of(text.as_str());
+        }
+
+        let text = SmolStr::new(self.buf.as_str());
+        self.buf.clear();
+        self.inner.token(kind, text);
+
+        // Add whitespace between adjoint puncts
+        let next = last.bump();
+        if let (
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))),
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))),
+        ) = (last.token_tree(), next.token_tree())
+        {
+            // Note: We always assume the semi-colon would be the last token in
+            // other parts of RA such that we don't add whitespace here.
+            if curr.spacing == tt::Spacing::Alone && curr.char != ';' {
+                self.inner.token(WHITESPACE, " ".into());
+                self.text_pos += TextSize::of(' ');
+            }
+        }
+    }
+
+    fn start_node(&mut self, kind: SyntaxKind) {
+        self.inner.start_node(kind);
+
+        match self.roots.last_mut() {
+            None | Some(0) => self.roots.push(1),
+            Some(ref mut n) => **n += 1,
+        };
+    }
+
+    fn finish_node(&mut self) {
+        self.inner.finish_node();
+        *self.roots.last_mut().unwrap() -= 1;
+    }
+
+    fn error(&mut self, error: ParseError) {
+        self.inner.error(error, self.text_pos)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::parse_macro;
+    use parser::TokenSource;
+    use syntax::{
+        algo::{insert_children, InsertPosition},
+        ast::AstNode,
+    };
+
+    #[test]
+    fn convert_tt_token_source() {
+        let expansion = parse_macro(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        )
+        .expand_tt("literals!(foo);");
+        let tts = &[expansion.into()];
+        let buffer = tt::buffer::TokenBuffer::new(tts);
+        let mut tt_src = SubtreeTokenSource::new(&buffer);
+        let mut tokens = vec![];
+        while tt_src.current().kind != EOF {
+            tokens.push((tt_src.current().kind, tt_src.text()));
+            tt_src.bump();
+        }
+
+        // [${]
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(tokens[2 + 3].1, "'c'");
+        assert_eq!(tokens[2 + 3].0, CHAR);
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(tokens[2 + 5 + 3].1, "1000");
+        assert_eq!(tokens[2 + 5 + 3].0, INT_NUMBER);
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(tokens[2 + 10 + 3].1, "12E+99_f64");
+        assert_eq!(tokens[2 + 10 + 3].0, FLOAT_NUMBER);
+
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(tokens[2 + 15 + 3].1, "\"rust1\"");
+        assert_eq!(tokens[2 + 15 + 3].0, STRING);
+    }
+
+    #[test]
+    fn stmts_token_trees_to_expr_is_err() {
+        let expansion = parse_macro(
+            r#"
+            macro_rules! stmts {
+                () => {
+                    let a = 0;
+                    let b = 0;
+                    let c = 0;
+                    let d = 0;
+                }
+            }
+            "#,
+        )
+        .expand_tt("stmts!();");
+        assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err());
+    }
+
+    #[test]
+    fn test_token_tree_last_child_is_white_space() {
+        let source_file = ast::SourceFile::parse("f!({} );").ok().unwrap();
+        let macro_call = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+        let token_tree = macro_call.token_tree().unwrap();
+
+        // Token Tree now is :
+        // TokenTree
+        // - T!['(']
+        // - TokenTree
+        //   - T!['{']
+        //   - T!['}']
+        // - WHITE_SPACE
+        // - T![')']
+
+        let rbrace =
+            token_tree.syntax().descendants_with_tokens().find(|it| it.kind() == T!['}']).unwrap();
+        let space = token_tree
+            .syntax()
+            .descendants_with_tokens()
+            .find(|it| it.kind() == SyntaxKind::WHITESPACE)
+            .unwrap();
+
+        // reorder th white space, such that the white is inside the inner token-tree.
+        let token_tree = insert_children(
+            &rbrace.parent().unwrap(),
+            InsertPosition::Last,
+            std::iter::once(space),
+        );
+
+        // Token Tree now is :
+        // TokenTree
+        // - T!['{']
+        // - T!['}']
+        // - WHITE_SPACE
+        let token_tree = ast::TokenTree::cast(token_tree).unwrap();
+        let tt = ast_to_token_tree(&token_tree).unwrap().0;
+
+        assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace));
+    }
+
+    #[test]
+    fn test_token_tree_multi_char_punct() {
+        let source_file = ast::SourceFile::parse("struct Foo { a: x::Y }").ok().unwrap();
+        let struct_def = source_file.syntax().descendants().find_map(ast::Struct::cast).unwrap();
+        let tt = ast_to_token_tree(&struct_def).unwrap().0;
+        token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap();
+    }
+}
diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs
new file mode 100644
index 000000000..0796ceee1
--- /dev/null
+++ b/crates/mbe/src/tests.rs
@@ -0,0 +1,1898 @@
+use std::fmt::Write;
+
+use ::parser::FragmentKind;
+use syntax::{ast, AstNode, NodeOrToken, SyntaxKind::IDENT, SyntaxNode, WalkEvent, T};
+use test_utils::assert_eq_text;
+
+use super::*;
+
+mod rule_parsing {
+    use syntax::{ast, AstNode};
+
+    use crate::ast_to_token_tree;
+
+    use super::*;
+
+    #[test]
+    fn test_valid_arms() {
+        fn check(macro_body: &str) {
+            let m = parse_macro_arm(macro_body);
+            m.unwrap();
+        }
+
+        check("($i:ident) => ()");
+        check("($($i:ident)*) => ($_)");
+        check("($($true:ident)*) => ($true)");
+        check("($($false:ident)*) => ($false)");
+        check("($) => ($)");
+    }
+
+    #[test]
+    fn test_invalid_arms() {
+        fn check(macro_body: &str, err: &str) {
+            let m = parse_macro_arm(macro_body);
+            assert_eq!(m, Err(ParseError::Expected(String::from(err))));
+        }
+
+        check("invalid", "expected subtree");
+
+        check("$i:ident => ()", "expected subtree");
+        check("($i:ident) ()", "expected `=`");
+        check("($($i:ident)_) => ()", "invalid repeat");
+
+        check("($i) => ($i)", "invalid macro definition");
+        check("($i:) => ($i)", "invalid macro definition");
+    }
+
+    fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> {
+        let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition);
+        let source_file = ast::SourceFile::parse(&macro_definition).ok().unwrap();
+        let macro_definition =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (definition_tt, _) =
+            ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
+        crate::MacroRules::parse(&definition_tt)
+    }
+}
+
+// Good first issue (although a slightly challenging one):
+//
+// * Pick a random test from here
+//   https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
+// * Port the test to rust and add it to this module
+// * Make it pass :-)
+
+#[test]
+fn test_token_id_shift() {
+    let expansion = parse_macro(
+        r#"
+macro_rules! foobar {
+    ($e:ident) => { foo bar $e }
+}
+"#,
+    )
+    .expand_tt("foobar!(baz);");
+
+    fn get_id(t: &tt::TokenTree) -> Option<u32> {
+        if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = t {
+            return Some(ident.id.0);
+        }
+        None
+    }
+
+    assert_eq!(expansion.token_trees.len(), 3);
+    // {($e:ident) => { foo bar $e }}
+    // 012345      67 8 9   T   12
+    assert_eq!(get_id(&expansion.token_trees[0]), Some(9));
+    assert_eq!(get_id(&expansion.token_trees[1]), Some(10));
+
+    // The input args of macro call include parentheses:
+    // (baz)
+    // So baz should be 12+1+1
+    assert_eq!(get_id(&expansion.token_trees[2]), Some(14));
+}
+
+#[test]
+fn test_token_map() {
+    let expanded = parse_macro(
+        r#"
+macro_rules! foobar {
+    ($e:ident) => { fn $e() {} }
+}
+"#,
+    )
+    .expand_tt("foobar!(baz);");
+
+    let (node, token_map) = token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap();
+    let content = node.syntax_node().to_string();
+
+    let get_text = |id, kind| -> String {
+        content[token_map.range_by_token(id).unwrap().by_kind(kind).unwrap()].to_string()
+    };
+
+    assert_eq!(expanded.token_trees.len(), 4);
+    // {($e:ident) => { fn $e() {} }}
+    // 012345      67 8 9  T12  3
+
+    assert_eq!(get_text(tt::TokenId(9), IDENT), "fn");
+    assert_eq!(get_text(tt::TokenId(12), T!['(']), "(");
+    assert_eq!(get_text(tt::TokenId(13), T!['{']), "{");
+}
+
+#[test]
+fn test_convert_tt() {
+    parse_macro(r#"
+macro_rules! impl_froms {
+    ($e:ident: $($v:ident),*) => {
+        $(
+            impl From<$v> for $e {
+                fn from(it: $v) -> $e {
+                    $e::$v(it)
+                }
+            }
+        )*
+    }
+}
+"#)
+        .assert_expand_tt(
+            "impl_froms!(TokenTree: Leaf, Subtree);",
+            "impl From <Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \
+             impl From <Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}"
+        );
+}
+
+#[test]
+fn test_convert_tt2() {
+    parse_macro(
+        r#"
+macro_rules! impl_froms {
+    ($e:ident: $($v:ident),*) => {
+        $(
+            impl From<$v> for $e {
+                fn from(it: $v) -> $e {
+                    $e::$v(it)
+                }
+            }
+        )*
+    }
+}
+"#,
+    )
+    .assert_expand(
+        "impl_froms!(TokenTree: Leaf, Subtree);",
+        r#"
+SUBTREE $
+  IDENT   impl 20
+  IDENT   From 21
+  PUNCH   < [joint] 22
+  IDENT   Leaf 53
+  PUNCH   > [alone] 25
+  IDENT   for 26
+  IDENT   TokenTree 51
+  SUBTREE {} 29
+    IDENT   fn 30
+    IDENT   from 31
+    SUBTREE () 32
+      IDENT   it 33
+      PUNCH   : [alone] 34
+      IDENT   Leaf 53
+    PUNCH   - [joint] 37
+    PUNCH   > [alone] 38
+    IDENT   TokenTree 51
+    SUBTREE {} 41
+      IDENT   TokenTree 51
+      PUNCH   : [joint] 44
+      PUNCH   : [joint] 45
+      IDENT   Leaf 53
+      SUBTREE () 48
+        IDENT   it 49
+  IDENT   impl 20
+  IDENT   From 21
+  PUNCH   < [joint] 22
+  IDENT   Subtree 55
+  PUNCH   > [alone] 25
+  IDENT   for 26
+  IDENT   TokenTree 51
+  SUBTREE {} 29
+    IDENT   fn 30
+    IDENT   from 31
+    SUBTREE () 32
+      IDENT   it 33
+      PUNCH   : [alone] 34
+      IDENT   Subtree 55
+    PUNCH   - [joint] 37
+    PUNCH   > [alone] 38
+    IDENT   TokenTree 51
+    SUBTREE {} 41
+      IDENT   TokenTree 51
+      PUNCH   : [joint] 44
+      PUNCH   : [joint] 45
+      IDENT   Subtree 55
+      SUBTREE () 48
+        IDENT   it 49
+"#,
+    );
+}
+
+#[test]
+fn test_lifetime_split() {
+    parse_macro(
+        r#"
+macro_rules! foo {
+    ($($t:tt)*) => { $($t)*}
+}
+"#,
+    )
+    .assert_expand(
+        r#"foo!(static bar: &'static str = "hello";);"#,
+        r#"
+SUBTREE $
+  IDENT   static 17
+  IDENT   bar 18
+  PUNCH   : [alone] 19
+  PUNCH   & [alone] 20
+  PUNCH   ' [joint] 21
+  IDENT   static 22
+  IDENT   str 23
+  PUNCH   = [alone] 24
+  LITERAL "hello" 25
+  PUNCH   ; [joint] 26
+"#,
+    );
+}
+
+#[test]
+fn test_expr_order() {
+    let expanded = parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:expr) => {
+                 fn bar() { $ i * 2; }
+            }
+        }
+"#,
+    )
+    .expand_items("foo! { 1 + 1}");
+
+    let dump = format!("{:#?}", expanded);
+    assert_eq_text!(
+        dump.trim(),
+        r#"[email protected]
+  [email protected]
+    [email protected] "fn"
+    [email protected]
+      [email protected] "bar"
+    [email protected]
+      [email protected] "("
+      [email protected] ")"
+    [email protected]
+      [email protected] "{"
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected] "1"
+            [email protected] "+"
+            [email protected]
+              [email protected] "1"
+          [email protected] "*"
+          [email protected]
+            [email protected] "2"
+        [email protected] ";"
+      [email protected] "}""#,
+    );
+}
+
+#[test]
+fn test_fail_match_pattern_by_first_token() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:ident) => (
+                mod $ i {}
+            );
+            (= $ i:ident) => (
+                fn $ i() {}
+            );
+            (+ $ i:ident) => (
+                struct $ i;
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo }", "mod foo {}")
+    .assert_expand_items("foo! { = bar }", "fn bar () {}")
+    .assert_expand_items("foo! { + Baz }", "struct Baz ;");
+}
+
+#[test]
+fn test_fail_match_pattern_by_last_token() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:ident) => (
+                mod $ i {}
+            );
+            ($ i:ident =) => (
+                fn $ i() {}
+            );
+            ($ i:ident +) => (
+                struct $ i;
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo }", "mod foo {}")
+    .assert_expand_items("foo! { bar = }", "fn bar () {}")
+    .assert_expand_items("foo! { Baz + }", "struct Baz ;");
+}
+
+#[test]
+fn test_fail_match_pattern_by_word_token() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:ident) => (
+                mod $ i {}
+            );
+            (spam $ i:ident) => (
+                fn $ i() {}
+            );
+            (eggs $ i:ident) => (
+                struct $ i;
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo }", "mod foo {}")
+    .assert_expand_items("foo! { spam bar }", "fn bar () {}")
+    .assert_expand_items("foo! { eggs Baz }", "struct Baz ;");
+}
+
+#[test]
+fn test_match_group_pattern_by_separator_token() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ ($ i:ident),*) => ($ (
+                mod $ i {}
+            )*);
+            ($ ($ i:ident)#*) => ($ (
+                fn $ i() {}
+            )*);
+            ($ i:ident ,# $ j:ident) => (
+                struct $ i;
+                struct $ j;
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo, bar }", "mod foo {} mod bar {}")
+    .assert_expand_items("foo! { foo# bar }", "fn foo () {} fn bar () {}")
+    .assert_expand_items("foo! { Foo,# Bar }", "struct Foo ; struct Bar ;");
+}
+
+#[test]
+fn test_match_group_pattern_with_multiple_defs() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ ($ i:ident),*) => ( struct Bar { $ (
+                fn $ i {}
+            )*} );
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo, bar }", "struct Bar {fn foo {} fn bar {}}");
+}
+
+#[test]
+fn test_match_group_pattern_with_multiple_statement() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ ($ i:ident),*) => ( fn baz { $ (
+                $ i ();
+            )*} );
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo, bar }", "fn baz {foo () ; bar () ;}");
+}
+
+#[test]
+fn test_match_group_pattern_with_multiple_statement_without_semi() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ ($ i:ident),*) => ( fn baz { $ (
+                $i()
+            );*} );
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo, bar }", "fn baz {foo () ;bar ()}");
+}
+
+#[test]
+fn test_match_group_empty_fixed_token() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ ($ i:ident)* #abc) => ( fn baz { $ (
+                $ i ();
+            )*} );
+        }
+"#,
+    )
+    .assert_expand_items("foo! {#abc}", "fn baz {}");
+}
+
+#[test]
+fn test_match_group_in_subtree() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            (fn $name:ident {$($i:ident)*} ) => ( fn $name() { $ (
+                $ i ();
+            )*} );
+        }"#,
+    )
+    .assert_expand_items("foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}");
+}
+
+#[test]
+fn test_match_group_with_multichar_sep() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} );
+        }"#,
+    )
+    .assert_expand_items("foo! (fn baz {true true} );", "fn baz () -> bool {true &&true}");
+}
+
+#[test]
+fn test_match_group_zero_match() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ( $($i:ident)* ) => ();
+        }"#,
+    )
+    .assert_expand_items("foo! ();", "");
+}
+
+#[test]
+fn test_match_group_in_group() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* );
+        }"#,
+    )
+    .assert_expand_items("foo! ( (a b) );", "(a b)");
+}
+
+#[test]
+fn test_expand_to_item_list() {
+    let tree = parse_macro(
+        "
+            macro_rules! structs {
+                ($($i:ident),*) => {
+                    $(struct $i { field: u32 } )*
+                }
+            }
+            ",
+    )
+    .expand_items("structs!(Foo, Bar);");
+    assert_eq!(
+        format!("{:#?}", tree).trim(),
+        r#"
+[email protected]
+  [email protected]
+    [email protected] "struct"
+    [email protected]
+      [email protected] "Foo"
+    [email protected]
+      [email protected] "{"
+      [email protected]
+        [email protected]
+          [email protected] "field"
+        [email protected] ":"
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected]
+                [email protected] "u32"
+      [email protected] "}"
+  [email protected]
+    [email protected] "struct"
+    [email protected]
+      [email protected] "Bar"
+    [email protected]
+      [email protected] "{"
+      [email protected]
+        [email protected]
+          [email protected] "field"
+        [email protected] ":"
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected]
+                [email protected] "u32"
+      [email protected] "}""#
+            .trim()
+    );
+}
+
+fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
+    if let tt::TokenTree::Subtree(subtree) = tt {
+        return &subtree;
+    }
+    unreachable!("It is not a subtree");
+}
+fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
+    if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
+        return lit;
+    }
+    unreachable!("It is not a literal");
+}
+
+fn to_punct(tt: &tt::TokenTree) -> &tt::Punct {
+    if let tt::TokenTree::Leaf(tt::Leaf::Punct(lit)) = tt {
+        return lit;
+    }
+    unreachable!("It is not a Punct");
+}
+
+#[test]
+fn test_expand_literals_to_token_tree() {
+    let expansion = parse_macro(
+        r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+    )
+    .expand_tt("literals!(foo);");
+    let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees;
+
+    // [let] [a] [=] ['c'] [;]
+    assert_eq!(to_literal(&stm_tokens[3]).text, "'c'");
+    // [let] [c] [=] [1000] [;]
+    assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000");
+    // [let] [f] [=] [12E+99_f64] [;]
+    assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64");
+    // [let] [s] [=] ["rust1"] [;]
+    assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
+}
+
+#[test]
+fn test_attr_to_token_tree() {
+    let expansion = parse_to_token_tree_by_syntax(
+        r#"
+            #[derive(Copy)]
+            struct Foo;
+            "#,
+    );
+
+    assert_eq!(to_punct(&expansion.token_trees[0]).char, '#');
+    assert_eq!(
+        to_subtree(&expansion.token_trees[1]).delimiter_kind(),
+        Some(tt::DelimiterKind::Bracket)
+    );
+}
+
+#[test]
+fn test_two_idents() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:ident, $ j:ident) => {
+                fn foo() { let a = $ i; let b = $j; }
+            }
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
+}
+
+#[test]
+fn test_tt_to_stmts() {
+    let stmts = parse_macro(
+        r#"
+        macro_rules! foo {
+            () => {
+                 let a = 0;
+                 a = 10 + 1;
+                 a
+            }
+        }
+"#,
+    )
+    .expand_statements("foo!{}");
+
+    assert_eq!(
+        format!("{:#?}", stmts).trim(),
+        r#"[email protected]
+  [email protected]
+    [email protected] "let"
+    [email protected]
+      [email protected]
+        [email protected] "a"
+    [email protected] "="
+    [email protected]
+      [email protected] "0"
+    [email protected] ";"
+  [email protected]
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected] "a"
+      [email protected] "="
+      [email protected]
+        [email protected]
+          [email protected] "10"
+        [email protected] "+"
+        [email protected]
+          [email protected] "1"
+    [email protected] ";"
+  [email protected]
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected] "a""#,
+    );
+}
+
+#[test]
+fn test_match_literal() {
+    parse_macro(
+        r#"
+    macro_rules! foo {
+        ('(') => {
+            fn foo() {}
+        }
+    }
+"#,
+    )
+    .assert_expand_items("foo! ['('];", "fn foo () {}");
+}
+
+// The following tests are port from intellij-rust directly
+// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
+
+#[test]
+fn test_path() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:path) => {
+                fn foo() { let a = $ i; }
+            }
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo }", "fn foo () {let a = foo ;}")
+    .assert_expand_items(
+        "foo! { bar::<u8>::baz::<u8> }",
+        "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}",
+    );
+}
+
+#[test]
+fn test_two_paths() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:path, $ j:path) => {
+                fn foo() { let a = $ i; let b = $j; }
+            }
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
+}
+
+#[test]
+fn test_path_with_path() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:path) => {
+                fn foo() { let a = $ i :: bar; }
+            }
+        }
+"#,
+    )
+    .assert_expand_items("foo! { foo }", "fn foo () {let a = foo :: bar ;}");
+}
+
+#[test]
+fn test_expr() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:expr) => {
+                 fn bar() { $ i; }
+            }
+        }
+"#,
+    )
+    .assert_expand_items(
+        "foo! { 2 + 2 * baz(3).quux() }",
+        "fn bar () {2 + 2 * baz (3) . quux () ;}",
+    );
+}
+
+#[test]
+fn test_last_expr() {
+    parse_macro(
+        r#"
+        macro_rules! vec {
+            ($($item:expr),*) => {
+                {
+                    let mut v = Vec::new();
+                    $(
+                        v.push($item);
+                    )*
+                    v
+                }
+            };
+        }
+"#,
+    )
+    .assert_expand_items(
+        "vec!(1,2,3);",
+        "{let mut v = Vec :: new () ; v . push (1) ; v . push (2) ; v . push (3) ; v}",
+    );
+}
+
+#[test]
+fn test_ty() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:ty) => (
+                fn bar() -> $ i { unimplemented!() }
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! { Baz<u8> }", "fn bar () -> Baz < u8 > {unimplemented ! ()}");
+}
+
+#[test]
+fn test_ty_with_complex_type() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:ty) => (
+                fn bar() -> $ i { unimplemented!() }
+            )
+        }
+"#,
+    )
+    // Reference lifetime struct with generic type
+    .assert_expand_items(
+        "foo! { &'a Baz<u8> }",
+        "fn bar () -> & 'a Baz < u8 > {unimplemented ! ()}",
+    )
+    // extern "Rust" func type
+    .assert_expand_items(
+        r#"foo! { extern "Rust" fn() -> Ret }"#,
+        r#"fn bar () -> extern "Rust" fn () -> Ret {unimplemented ! ()}"#,
+    );
+}
+
+#[test]
+fn test_pat_() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:pat) => { fn foo() { let $ i; } }
+        }
+"#,
+    )
+    .assert_expand_items("foo! { (a, b) }", "fn foo () {let (a , b) ;}");
+}
+
+#[test]
+fn test_stmt() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:stmt) => (
+                fn bar() { $ i; }
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! { 2 }", "fn bar () {2 ;}")
+    .assert_expand_items("foo! { let a = 0 }", "fn bar () {let a = 0 ;}");
+}
+
+#[test]
+fn test_single_item() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:item) => (
+                $ i
+            )
+        }
+"#,
+    )
+    .assert_expand_items("foo! {mod c {}}", "mod c {}");
+}
+
+#[test]
+fn test_all_items() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ ($ i:item)*) => ($ (
+                $ i
+            )*)
+        }
+"#,
+    ).
+    assert_expand_items(
+        r#"
+        foo! {
+            extern crate a;
+            mod b;
+            mod c {}
+            use d;
+            const E: i32 = 0;
+            static F: i32 = 0;
+            impl G {}
+            struct H;
+            enum I { Foo }
+            trait J {}
+            fn h() {}
+            extern {}
+            type T = u8;
+        }
+"#,
+        r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#,
+    );
+}
+
+#[test]
+fn test_block() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:block) => { fn foo() $ i }
+        }
+"#,
+    )
+    .assert_expand_statements("foo! { { 1; } }", "fn foo () {1 ;}");
+}
+
+#[test]
+fn test_meta() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($ i:meta) => (
+                #[$ i]
+                fn bar() {}
+            )
+        }
+"#,
+    )
+    .assert_expand_items(
+        r#"foo! { cfg(target_os = "windows") }"#,
+        r#"# [cfg (target_os = "windows")] fn bar () {}"#,
+    );
+}
+
+#[test]
+fn test_meta_doc_comments() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($(#[$ i:meta])+) => (
+                $(#[$ i])+
+                fn bar() {}
+            )
+        }
+"#,
+    ).
+    assert_expand_items(
+        r#"foo! {
+            /// Single Line Doc 1
+            /**
+                MultiLines Doc
+            */
+        }"#,
+        "# [doc = \" Single Line Doc 1\"] # [doc = \"\\\\n                MultiLines Doc\\\\n            \"] fn bar () {}",
+    );
+}
+
+#[test]
+fn test_tt_block() {
+    parse_macro(
+        r#"
+            macro_rules! foo {
+                ($ i:tt) => { fn foo() $ i }
+            }
+    "#,
+    )
+    .assert_expand_items(r#"foo! { { 1; } }"#, r#"fn foo () {1 ;}"#);
+}
+
+#[test]
+fn test_tt_group() {
+    parse_macro(
+        r#"
+            macro_rules! foo {
+                 ($($ i:tt)*) => { $($ i)* }
+            }
+    "#,
+    )
+    .assert_expand_items(r#"foo! { fn foo() {} }"#, r#"fn foo () {}"#);
+}
+
+#[test]
+fn test_tt_composite() {
+    parse_macro(
+        r#"
+            macro_rules! foo {
+                 ($i:tt) => { 0 }
+            }
+    "#,
+    )
+    .assert_expand_items(r#"foo! { => }"#, r#"0"#);
+}
+
+#[test]
+fn test_tt_composite2() {
+    let node = parse_macro(
+        r#"
+            macro_rules! foo {
+                ($($tt:tt)*) => { abs!(=> $($tt)*) }
+            }
+    "#,
+    )
+    .expand_items(r#"foo!{#}"#);
+
+    let res = format!("{:#?}", &node);
+    assert_eq_text!(
+        res.trim(),
+        r###"[email protected]
+  [email protected]
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected] "abs"
+    [email protected] "!"
+    [email protected]
+      [email protected] "("
+      [email protected] "="
+      [email protected] ">"
+      [email protected] " "
+      [email protected] "#"
+      [email protected] ")""###
+    );
+}
+
+#[test]
+fn test_lifetime() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+              ($ lt:lifetime) => { struct Ref<$ lt>{ s: &$ lt str } }
+        }
+"#,
+    )
+    .assert_expand_items(r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#);
+}
+
+#[test]
+fn test_literal() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+              ($ type:ty $ lit:literal) => { const VALUE: $ type = $ lit;};
+        }
+"#,
+    )
+    .assert_expand_items(r#"foo!(u8 0);"#, r#"const VALUE : u8 = 0 ;"#);
+}
+
+#[test]
+fn test_boolean_is_ident() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+              ($lit0:literal, $lit1:literal) => { const VALUE: (bool,bool) = ($lit0,$lit1); };
+        }
+"#,
+    )
+    .assert_expand(
+        r#"foo!(true,false);"#,
+        r#"
+SUBTREE $
+  IDENT   const 14
+  IDENT   VALUE 15
+  PUNCH   : [alone] 16
+  SUBTREE () 17
+    IDENT   bool 18
+    PUNCH   , [alone] 19
+    IDENT   bool 20
+  PUNCH   = [alone] 21
+  SUBTREE () 22
+    IDENT   true 29
+    PUNCH   , [joint] 25
+    IDENT   false 31
+  PUNCH   ; [alone] 28
+"#,
+    );
+}
+
+#[test]
+fn test_vis() {
+    parse_macro(
+        r#"
+        macro_rules! foo {
+              ($ vis:vis $ name:ident) => { $ vis fn $ name() {}};
+        }
+"#,
+    )
+    .assert_expand_items(r#"foo!(pub foo);"#, r#"pub fn foo () {}"#)
+    // test optional cases
+    .assert_expand_items(r#"foo!(foo);"#, r#"fn foo () {}"#);
+}
+
+#[test]
+fn test_inner_macro_rules() {
+    parse_macro(
+        r#"
+macro_rules! foo {
+    ($a:ident, $b:ident, $c:tt) => {
+
+        macro_rules! bar {
+            ($bi:ident) => {
+                fn $bi() -> u8 {$c}
+            }
+        }
+
+        bar!($a);
+        fn $b() -> u8 {$c}
+    }
+}
+"#,
+    ).
+    assert_expand_items(
+        r#"foo!(x,y, 1);"#,
+        r#"macro_rules ! bar {($ bi : ident) => {fn $ bi () -> u8 {1}}} bar ! (x) ; fn y () -> u8 {1}"#,
+    );
+}
+
+// The following tests are based on real world situations
+#[test]
+fn test_vec() {
+    let fixture = parse_macro(
+        r#"
+         macro_rules! vec {
+            ($($item:expr),*) => {
+                {
+                    let mut v = Vec::new();
+                    $(
+                        v.push($item);
+                    )*
+                    v
+                }
+            };
+}
+"#,
+    );
+    fixture
+        .assert_expand_items(r#"vec!();"#, r#"{let mut v = Vec :: new () ; v}"#)
+        .assert_expand_items(
+            r#"vec![1u32,2];"#,
+            r#"{let mut v = Vec :: new () ; v . push (1u32) ; v . push (2) ; v}"#,
+        );
+
+    let tree = fixture.expand_expr(r#"vec![1u32,2];"#);
+
+    assert_eq!(
+        format!("{:#?}", tree).trim(),
+        r#"[email protected]
+  [email protected] "{"
+  [email protected]
+    [email protected] "let"
+    [email protected]
+      [email protected] "mut"
+      [email protected]
+        [email protected] "v"
+    [email protected] "="
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected]
+                [email protected] "Vec"
+          [email protected] "::"
+          [email protected]
+            [email protected]
+              [email protected] "new"
+      [email protected]
+        [email protected] "("
+        [email protected] ")"
+    [email protected] ";"
+  [email protected]
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected] "v"
+      [email protected] "."
+      [email protected]
+        [email protected] "push"
+      [email protected]
+        [email protected] "("
+        [email protected]
+          [email protected] "1u32"
+        [email protected] ")"
+    [email protected] ";"
+  [email protected]
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected]
+              [email protected] "v"
+      [email protected] "."
+      [email protected]
+        [email protected] "push"
+      [email protected]
+        [email protected] "("
+        [email protected]
+          [email protected] "2"
+        [email protected] ")"
+    [email protected] ";"
+  [email protected]
+    [email protected]
+      [email protected]
+        [email protected]
+          [email protected] "v"
+  [email protected] "}""#
+    );
+}
+
+#[test]
+fn test_winapi_struct() {
+    // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/macros.rs#L366
+
+    parse_macro(
+        r#"
+macro_rules! STRUCT {
+    ($(#[$attrs:meta])* struct $name:ident {
+        $($field:ident: $ftype:ty,)+
+    }) => (
+        #[repr(C)] #[derive(Copy)] $(#[$attrs])*
+        pub struct $name {
+            $(pub $field: $ftype,)+
+        }
+        impl Clone for $name {
+            #[inline]
+            fn clone(&self) -> $name { *self }
+        }
+        #[cfg(feature = "impl-default")]
+        impl Default for $name {
+            #[inline]
+            fn default() -> $name { unsafe { $crate::_core::mem::zeroed() } }
+        }
+    );
+}
+"#,
+    ).
+    // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/shared/d3d9caps.rs
+    assert_expand_items(r#"STRUCT!{struct D3DVSHADERCAPS2_0 {Caps: u8,}}"#,
+        "# [repr (C)] # [derive (Copy)] pub struct D3DVSHADERCAPS2_0 {pub Caps : u8 ,} impl Clone for D3DVSHADERCAPS2_0 {# [inline] fn clone (& self) -> D3DVSHADERCAPS2_0 {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DVSHADERCAPS2_0 {# [inline] fn default () -> D3DVSHADERCAPS2_0 {unsafe {$crate :: _core :: mem :: zeroed ()}}}"
+    )
+    .assert_expand_items(r#"STRUCT!{#[cfg_attr(target_arch = "x86", repr(packed))] struct D3DCONTENTPROTECTIONCAPS {Caps : u8 ,}}"#,
+        "# [repr (C)] # [derive (Copy)] # [cfg_attr (target_arch = \"x86\" , repr (packed))] pub struct D3DCONTENTPROTECTIONCAPS {pub Caps : u8 ,} impl Clone for D3DCONTENTPROTECTIONCAPS {# [inline] fn clone (& self) -> D3DCONTENTPROTECTIONCAPS {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DCONTENTPROTECTIONCAPS {# [inline] fn default () -> D3DCONTENTPROTECTIONCAPS {unsafe {$crate :: _core :: mem :: zeroed ()}}}"
+    );
+}
+
+#[test]
+fn test_int_base() {
+    parse_macro(
+        r#"
+macro_rules! int_base {
+    ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => {
+        #[stable(feature = "rust1", since = "1.0.0")]
+        impl fmt::$Trait for $T {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                $Radix.fmt_int(*self as $U, f)
+            }
+        }
+    }
+}
+"#,
+    ).assert_expand_items(r#" int_base!{Binary for isize as usize -> Binary}"#,
+        "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
+    );
+}
+
+#[test]
+fn test_generate_pattern_iterators() {
+    // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs
+    parse_macro(
+        r#"
+macro_rules! generate_pattern_iterators {
+        { double ended; with $(#[$common_stability_attribute:meta])*,
+                           $forward_iterator:ident,
+                           $reverse_iterator:ident, $iterty:ty
+        } => {
+            fn foo(){}
+        }
+}
+"#,
+    ).assert_expand_items(
+        r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#,
+        "fn foo () {}",
+    );
+}
+
+#[test]
+fn test_impl_fn_for_zst() {
+    // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs
+    parse_macro(
+        r#"
+macro_rules! impl_fn_for_zst  {
+        {  $( $( #[$attr: meta] )*
+        struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
+            |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty
+$body: block; )+
+        } => {
+           $(
+            $( #[$attr] )*
+            struct $Name;
+
+            impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name {
+                #[inline]
+                extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
+                    $body
+                }
+            }
+
+            impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name {
+                #[inline]
+                extern "rust-call" fn call_mut(
+                    &mut self,
+                    ($( $arg, )*): ($( $ArgTy, )*)
+                ) -> $ReturnTy {
+                    Fn::call(&*self, ($( $arg, )*))
+                }
+            }
+
+            impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name {
+                type Output = $ReturnTy;
+
+                #[inline]
+                extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
+                    Fn::call(&self, ($( $arg, )*))
+                }
+            }
+        )+
+}
+        }
+"#,
+    ).assert_expand_items(r#"
+impl_fn_for_zst !   {
+     # [ derive ( Clone ) ]
+     struct   CharEscapeDebugContinue   impl   Fn   =   | c :   char |   ->   char :: EscapeDebug   {
+         c . escape_debug_ext ( false )
+     } ;
+
+     # [ derive ( Clone ) ]
+     struct   CharEscapeUnicode   impl   Fn   =   | c :   char |   ->   char :: EscapeUnicode   {
+         c . escape_unicode ( )
+     } ;
+     # [ derive ( Clone ) ]
+     struct   CharEscapeDefault   impl   Fn   =   | c :   char |   ->   char :: EscapeDefault   {
+         c . escape_default ( )
+     } ;
+ }
+"#,
+        "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}"
+    );
+}
+
+#[test]
+fn test_impl_nonzero_fmt() {
+    // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
+    parse_macro(
+        r#"
+        macro_rules! impl_nonzero_fmt {
+            ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
+                fn foo () {}
+            }
+        }
+"#,
+    ).assert_expand_items(
+        r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#,
+        "fn foo () {}",
+    );
+}
+
+#[test]
+fn test_cfg_if_items() {
+    // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986
+    parse_macro(
+        r#"
+        macro_rules! __cfg_if_items {
+            (($($not:meta,)*) ; ) => {};
+            (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => {
+                 __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* }
+            }
+        }
+"#,
+    ).assert_expand_items(
+        r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#,
+        "__cfg_if_items ! {(rustdoc ,) ;}",
+    );
+}
+
+#[test]
+fn test_cfg_if_main() {
+    // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9
+    parse_macro(
+        r#"
+        macro_rules! cfg_if {
+            ($(
+                if #[cfg($($meta:meta),*)] { $($it:item)* }
+            ) else * else {
+                $($it2:item)*
+            }) => {
+                __cfg_if_items! {
+                    () ;
+                    $( ( ($($meta),*) ($($it)*) ), )*
+                    ( () ($($it2)*) ),
+                }
+            };
+
+            // Internal macro to Apply a cfg attribute to a list of items
+            (@__apply $m:meta, $($it:item)*) => {
+                $(#[$m] $it)*
+            };
+        }
+"#,
+    ).assert_expand_items(r#"
+cfg_if !   {
+     if   # [ cfg ( target_env   =   "msvc" ) ]   {
+         // no extra unwinder support needed
+     }   else   if   # [ cfg ( all ( target_arch   =   "wasm32" ,   not ( target_os   =   "emscripten" ) ) ) ]   {
+         // no unwinder on the system!
+     }   else   {
+         mod   libunwind ;
+         pub   use   libunwind :: * ;
+     }
+ }
+"#,
+        "__cfg_if_items ! {() ; ((target_env = \"msvc\") ()) , ((all (target_arch = \"wasm32\" , not (target_os = \"emscripten\"))) ()) , (() (mod libunwind ; pub use libunwind :: * ;)) ,}"
+    ).assert_expand_items(
+        r#"
+cfg_if ! { @ __apply cfg ( all ( not ( any ( not ( any ( target_os = "solaris" , target_os = "illumos" ) ) ) ) ) ) , }
+"#,
+        "",
+    );
+}
+
+#[test]
+fn test_proptest_arbitrary() {
+    // from https://github.com/AltSysrq/proptest/blob/d1c4b049337d2f75dd6f49a095115f7c532e5129/proptest/src/arbitrary/macros.rs#L16
+    parse_macro(
+        r#"
+macro_rules! arbitrary {
+    ([$($bounds : tt)*] $typ: ty, $strat: ty, $params: ty;
+        $args: ident => $logic: expr) => {
+        impl<$($bounds)*> $crate::arbitrary::Arbitrary for $typ {
+            type Parameters = $params;
+            type Strategy = $strat;
+            fn arbitrary_with($args: Self::Parameters) -> Self::Strategy {
+                $logic
+            }
+        }
+    };
+
+}"#,
+    ).assert_expand_items(r#"arbitrary !   ( [ A : Arbitrary ]
+        Vec < A > ,
+        VecStrategy < A :: Strategy > ,
+        RangedParams1 < A :: Parameters > ;
+        args =>   { let product_unpack !   [ range , a ] = args ; vec ( any_with :: < A >   ( a ) , range ) }
+    ) ;"#,
+    "impl <A : Arbitrary > $crate :: arbitrary :: Arbitrary for Vec < A > {type Parameters = RangedParams1 < A :: Parameters > ; type Strategy = VecStrategy < A :: Strategy > ; fn arbitrary_with (args : Self :: Parameters) -> Self :: Strategy {{let product_unpack ! [range , a] = args ; vec (any_with :: < A > (a) , range)}}}"
+    );
+}
+
+#[test]
+fn test_old_ridl() {
+    // This is from winapi 2.8, which do not have a link from github
+    //
+    let expanded = parse_macro(
+        r#"
+#[macro_export]
+macro_rules! RIDL {
+    (interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident)
+        {$(
+            fn $method:ident(&mut self $(,$p:ident : $t:ty)*) -> $rtr:ty
+        ),+}
+    ) => {
+        impl $interface {
+            $(pub unsafe fn $method(&mut self) -> $rtr {
+                ((*self.lpVtbl).$method)(self $(,$p)*)
+            })+
+        }
+    };
+}"#,
+    ).expand_tt(r#"
+    RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID3D11DeviceChildVtbl) {
+        fn GetDataSize(&mut self) -> UINT
+    }}"#);
+
+    assert_eq!(expanded.to_string(), "impl ID3D11Asynchronous {pub unsafe fn GetDataSize (& mut self) -> UINT {((* self . lpVtbl) .GetDataSize) (self)}}");
+}
+
+#[test]
+fn test_quick_error() {
+    let expanded = parse_macro(
+        r#"
+macro_rules! quick_error {
+
+ (SORT [enum $name:ident $( #[$meta:meta] )*]
+        items [$($( #[$imeta:meta] )*
+                  => $iitem:ident: $imode:tt [$( $ivar:ident: $ityp:ty ),*]
+                                {$( $ifuncs:tt )*} )* ]
+        buf [ ]
+        queue [ ]
+    ) => {
+        quick_error!(ENUMINITION [enum $name $( #[$meta] )*]
+            body []
+            queue [$(
+                $( #[$imeta] )*
+                =>
+                $iitem: $imode [$( $ivar: $ityp ),*]
+            )*]
+        );
+};
+
+}
+"#,
+    )
+    .expand_tt(
+        r#"
+quick_error ! (SORT [enum Wrapped # [derive (Debug)]] items [
+        => One : UNIT [] {}
+        => Two : TUPLE [s :String] {display ("two: {}" , s) from ()}
+    ] buf [] queue []) ;
+"#,
+    );
+
+    assert_eq!(expanded.to_string(), "quick_error ! (ENUMINITION [enum Wrapped # [derive (Debug)]] body [] queue [=> One : UNIT [] => Two : TUPLE [s : String]]) ;");
+}
+
+#[test]
+fn test_empty_repeat_vars_in_empty_repeat_vars() {
+    parse_macro(
+        r#"
+macro_rules! delegate_impl {
+    ([$self_type:ident, $self_wrap:ty, $self_map:ident]
+     pub trait $name:ident $(: $sup:ident)* $(+ $more_sup:ident)* {
+
+        // "Escaped" associated types. Stripped before making the `trait`
+        // itself, but forwarded when delegating impls.
+        $(
+        @escape [type $assoc_name_ext:ident]
+        // Associated types. Forwarded.
+        )*
+        $(
+        @section type
+        $(
+            $(#[$_assoc_attr:meta])*
+            type $assoc_name:ident $(: $assoc_bound:ty)*;
+        )+
+        )*
+        // Methods. Forwarded. Using $self_map!(self) around the self argument.
+        // Methods must use receiver `self` or explicit type like `self: &Self`
+        // &self and &mut self are _not_ supported.
+        $(
+        @section self
+        $(
+            $(#[$_method_attr:meta])*
+            fn $method_name:ident(self $(: $self_selftype:ty)* $(,$marg:ident : $marg_ty:ty)*) -> $mret:ty;
+        )+
+        )*
+        // Arbitrary tail that is ignored when forwarding.
+        $(
+        @section nodelegate
+        $($tail:tt)*
+        )*
+    }) => {
+        impl<> $name for $self_wrap where $self_type: $name {
+            $(
+            $(
+                fn $method_name(self $(: $self_selftype)* $(,$marg: $marg_ty)*) -> $mret {
+                    $self_map!(self).$method_name($($marg),*)
+                }
+            )*
+            )*
+        }
+    }
+}
+"#,
+    ).assert_expand_items(
+        r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#,
+        "impl <> Data for & \'a mut G where G : Data {}",
+    );
+}
+
+#[test]
+fn expr_interpolation() {
+    let expanded = parse_macro(
+        r#"
+        macro_rules! id {
+            ($expr:expr) => {
+                map($expr)
+            }
+        }
+        "#,
+    )
+    .expand_expr("id!(x + foo);");
+
+    assert_eq!(expanded.to_string(), "map(x+foo)");
+}
+
+pub(crate) struct MacroFixture {
+    rules: MacroRules,
+}
+
+impl MacroFixture {
+    pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree {
+        self.try_expand_tt(invocation).unwrap()
+    }
+
+    fn try_expand_tt(&self, invocation: &str) -> Result<tt::Subtree, ExpandError> {
+        let source_file = ast::SourceFile::parse(invocation).tree();
+        let macro_invocation =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
+            .ok_or_else(|| ExpandError::ConversionError)?;
+
+        self.rules.expand(&invocation_tt).result()
+    }
+
+    fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {
+        assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err));
+    }
+
+    fn expand_items(&self, invocation: &str) -> SyntaxNode {
+        let expanded = self.expand_tt(invocation);
+        token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node()
+    }
+
+    fn expand_statements(&self, invocation: &str) -> SyntaxNode {
+        let expanded = self.expand_tt(invocation);
+        token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node()
+    }
+
+    fn expand_expr(&self, invocation: &str) -> SyntaxNode {
+        let expanded = self.expand_tt(invocation);
+        token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node()
+    }
+
+    fn assert_expand_tt(&self, invocation: &str, expected: &str) {
+        let expansion = self.expand_tt(invocation);
+        assert_eq!(expansion.to_string(), expected);
+    }
+
+    fn assert_expand(&self, invocation: &str, expected: &str) {
+        let expansion = self.expand_tt(invocation);
+        let actual = format!("{:?}", expansion);
+        test_utils::assert_eq_text!(&actual.trim(), &expected.trim());
+    }
+
+    fn assert_expand_items(&self, invocation: &str, expected: &str) -> &MacroFixture {
+        self.assert_expansion(FragmentKind::Items, invocation, expected);
+        self
+    }
+
+    fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &MacroFixture {
+        self.assert_expansion(FragmentKind::Statements, invocation, expected);
+        self
+    }
+
+    fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) {
+        let expanded = self.expand_tt(invocation);
+        assert_eq!(expanded.to_string(), expected);
+
+        let expected = expected.replace("$crate", "C_C__C");
+
+        // wrap the given text to a macro call
+        let expected = {
+            let wrapped = format!("wrap_macro!( {} )", expected);
+            let wrapped = ast::SourceFile::parse(&wrapped);
+            let wrapped =
+                wrapped.tree().syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
+            let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0;
+            wrapped.delimiter = None;
+            wrapped
+        };
+
+        let expanded_tree = token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node();
+        let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string();
+
+        let expected_tree = token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node();
+        let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string();
+
+        let expected_tree = expected_tree.replace("C_C__C", "$crate");
+        assert_eq!(
+            expanded_tree, expected_tree,
+            "\nleft:\n{}\nright:\n{}",
+            expanded_tree, expected_tree,
+        );
+    }
+}
+
+fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree {
+    let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
+    let macro_definition =
+        source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+    let (definition_tt, _) = ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
+
+    let parsed = parse_to_token_tree(
+        &ra_fixture[macro_definition.token_tree().unwrap().syntax().text_range()],
+    )
+    .unwrap()
+    .0;
+    assert_eq!(definition_tt, parsed);
+
+    definition_tt
+}
+
+pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture {
+    let definition_tt = parse_macro_to_tt(ra_fixture);
+    let rules = MacroRules::parse(&definition_tt).unwrap();
+    MacroFixture { rules }
+}
+
+pub(crate) fn parse_macro_error(ra_fixture: &str) -> ParseError {
+    let definition_tt = parse_macro_to_tt(ra_fixture);
+
+    match MacroRules::parse(&definition_tt) {
+        Ok(_) => panic!("Expect error"),
+        Err(err) => err,
+    }
+}
+
+pub(crate) fn parse_to_token_tree_by_syntax(ra_fixture: &str) -> tt::Subtree {
+    let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap();
+    let tt = syntax_node_to_token_tree(source_file.syntax()).unwrap().0;
+
+    let parsed = parse_to_token_tree(ra_fixture).unwrap().0;
+    assert_eq!(tt, parsed);
+
+    parsed
+}
+
+fn debug_dump_ignore_spaces(node: &syntax::SyntaxNode) -> String {
+    let mut level = 0;
+    let mut buf = String::new();
+    macro_rules! indent {
+        () => {
+            for _ in 0..level {
+                buf.push_str("  ");
+            }
+        };
+    }
+
+    for event in node.preorder_with_tokens() {
+        match event {
+            WalkEvent::Enter(element) => {
+                match element {
+                    NodeOrToken::Node(node) => {
+                        indent!();
+                        writeln!(buf, "{:?}", node.kind()).unwrap();
+                    }
+                    NodeOrToken::Token(token) => match token.kind() {
+                        syntax::SyntaxKind::WHITESPACE => {}
+                        _ => {
+                            indent!();
+                            writeln!(buf, "{:?}", token.kind()).unwrap();
+                        }
+                    },
+                }
+                level += 1;
+            }
+            WalkEvent::Leave(_) => level -= 1,
+        }
+    }
+
+    buf
+}
+
+#[test]
+fn test_issue_2520() {
+    let macro_fixture = parse_macro(
+        r#"
+        macro_rules! my_macro {
+            {
+                ( $(
+                    $( [] $sname:ident : $stype:ty  )?
+                    $( [$expr:expr] $nname:ident : $ntype:ty  )?
+                ),* )
+            } => {
+                Test {
+                    $(
+                        $( $sname, )?
+                    )*
+                }
+            };
+        }
+    "#,
+    );
+
+    macro_fixture.assert_expand_items(
+        r#"my_macro ! {
+            ([] p1 : u32 , [|_| S0K0] s : S0K0 , [] k0 : i32)
+        }"#,
+        "Test {p1 , k0 ,}",
+    );
+}
+
+#[test]
+fn test_issue_3861() {
+    let macro_fixture = parse_macro(
+        r#"
+        macro_rules! rgb_color {
+            ($p:expr, $t: ty) => {
+                pub fn new() {
+                    let _ = 0 as $t << $p;
+                }
+            };
+        }
+    "#,
+    );
+
+    macro_fixture.expand_items(r#"rgb_color!(8 + 8, u32);"#);
+}
+
+#[test]
+fn test_repeat_bad_var() {
+    // FIXME: the second rule of the macro should be removed and an error about
+    // `$( $c )+` raised
+    parse_macro(
+        r#"
+        macro_rules! foo {
+            ($( $b:ident )+) => {
+                $( $c )+
+            };
+            ($( $b:ident )+) => {
+                $( $b )+
+            }
+        }
+    "#,
+    )
+    .assert_expand_items("foo!(b0 b1);", "b0 b1");
+}
+
+#[test]
+fn test_no_space_after_semi_colon() {
+    let expanded = parse_macro(
+        r#"
+        macro_rules! with_std { ($($i:item)*) => ($(#[cfg(feature = "std")]$i)*) }
+    "#,
+    )
+    .expand_items(r#"with_std! {mod m;mod f;}"#);
+
+    let dump = format!("{:#?}", expanded);
+    assert_eq_text!(
+        dump.trim(),
+        r###"[email protected]
+  [email protected]
+    [email protected]
+      [email protected] "#"
+      [email protected] "["
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected] "cfg"
+      [email protected]
+        [email protected] "("
+        [email protected] "feature"
+        [email protected] "="
+        [email protected] "\"std\""
+        [email protected] ")"
+      [email protected] "]"
+    [email protected] "mod"
+    [email protected]
+      [email protected] "m"
+    [email protected] ";"
+  [email protected]
+    [email protected]
+      [email protected] "#"
+      [email protected] "["
+      [email protected]
+        [email protected]
+          [email protected]
+            [email protected] "cfg"
+      [email protected]
+        [email protected] "("
+        [email protected] "feature"
+        [email protected] "="
+        [email protected] "\"std\""
+        [email protected] ")"
+      [email protected] "]"
+    [email protected] "mod"
+    [email protected]
+      [email protected] "f"
+    [email protected] ";""###,
+    );
+}
+
+// https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs
+#[test]
+fn test_rustc_issue_57597() {
+    fn test_error(fixture: &str) {
+        assert_eq!(parse_macro_error(fixture), ParseError::RepetitionEmtpyTokenTree);
+    }
+
+    test_error("macro_rules! foo { ($($($i:ident)?)+) => {}; }");
+    test_error("macro_rules! foo { ($($($i:ident)?)*) => {}; }");
+    test_error("macro_rules! foo { ($($($i:ident)?)?) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)?)?)?) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)*)?)?) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)?)*)?) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)?)?)*) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)*)*)?) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)?)*)*) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)?)*)+) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)+)?)*) => {}; }");
+    test_error("macro_rules! foo { ($($($($i:ident)+)*)?) => {}; }");
+}
+
+#[test]
+fn test_expand_bad_literal() {
+    parse_macro(
+        r#"
+        macro_rules! foo { ($i:literal) => {}; }
+    "#,
+    )
+    .assert_expand_err(r#"foo!(&k");"#, &ExpandError::BindingError("".into()));
+}
+
+#[test]
+fn test_empty_comments() {
+    parse_macro(
+        r#"
+        macro_rules! one_arg_macro { ($fmt:expr) => (); }
+    "#,
+    )
+    .assert_expand_err(
+        r#"one_arg_macro!(/**/)"#,
+        &ExpandError::BindingError("expected Expr".into()),
+    );
+}
diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs
new file mode 100644
index 000000000..46c420718
--- /dev/null
+++ b/crates/mbe/src/tt_iter.rs
@@ -0,0 +1,75 @@
+//! FIXME: write short doc here
+
+#[derive(Debug, Clone)]
+pub(crate) struct TtIter<'a> {
+    pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
+}
+
+impl<'a> TtIter<'a> {
+    pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
+        TtIter { inner: subtree.token_trees.iter() }
+    }
+
+    pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> {
+        match self.next() {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => {
+                Ok(())
+            }
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
+        match self.next() {
+            Some(tt::TokenTree::Subtree(it)) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
+        match self.next() {
+            Some(tt::TokenTree::Leaf(it)) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Ident(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> {
+        let it = self.expect_leaf()?;
+        match it {
+            tt::Leaf::Literal(_) => Ok(it),
+            tt::Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Punct(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> {
+        self.inner.as_slice().get(n)
+    }
+}
+
+impl<'a> Iterator for TtIter<'a> {
+    type Item = &'a tt::TokenTree;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next()
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
+impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {}