Merge #1858

1858: use usual token tree for macro expansions r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

2 files changed, 388 insertions, 261 deletions

diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 100a3b0e0..aff953102 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -1,11 +1,14 @@
 use crate::{
     mbe_expander::{Binding, Bindings, Fragment},
-    tt_cursor::TtCursor,
+    parser::{parse_pattern, Op, RepeatKind, Separator},
+    subtree_source::SubtreeTokenSource,
+    tt_iter::TtIter,
     ExpandError,
 };
 
-use ra_parser::FragmentKind::*;
-use ra_syntax::SmolStr;
+use ra_parser::{FragmentKind::*, TreeSink};
+use ra_syntax::{SmolStr, SyntaxKind};
+use tt::buffer::{Cursor, TokenBuffer};
 
 impl Bindings {
     fn push_optional(&mut self, name: &SmolStr) {
@@ -42,121 +45,247 @@ impl Bindings {
         }
         Ok(())
     }
+}
 
-    fn merge(&mut self, nested: Bindings) {
-        self.inner.extend(nested.inner);
-    }
+macro_rules! err {
+    () => {
+        ExpandError::BindingError(format!(""))
+    };
+    ($($tt:tt)*) => {
+        ExpandError::BindingError(format!($($tt)*))
+    };
 }
 
-pub(super) fn match_lhs(
-    pattern: &crate::Subtree,
-    input: &mut TtCursor,
-) -> Result<Bindings, ExpandError> {
+macro_rules! bail {
+    ($($tt:tt)*) => {
+        return Err(err!($($tt)*))
+    };
+}
+
+pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Bindings, ExpandError> {
+    assert!(pattern.delimiter == tt::Delimiter::None);
+
     let mut res = Bindings::default();
-    for pat in pattern.token_trees.iter() {
-        match pat {
-            crate::TokenTree::Leaf(leaf) => match leaf {
-                crate::Leaf::Var(crate::Var { text, kind }) => {
-                    let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
-                    match match_meta_var(kind.as_str(), input)? {
-                        Some(fragment) => {
-                            res.inner.insert(text.clone(), Binding::Fragment(fragment));
-                        }
-                        None => res.push_optional(text),
-                    }
+    let mut src = TtIter::new(src);
+
+    match_subtree(&mut res, pattern, &mut src)?;
+
+    if src.len() > 0 {
+        bail!("leftover tokens");
+    }
+
+    Ok(res)
+}
+
+fn match_subtree(
+    bindings: &mut Bindings,
+    pattern: &tt::Subtree,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::TokenTree(tt::TokenTree::Leaf(lhs)) => {
+                let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?;
+                match (lhs, rhs) {
+                    (
+                        tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
+                        tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
+                        tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
+                        tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    _ => Err(ExpandError::UnexpectedToken)?,
                 }
-                crate::Leaf::Punct(punct) => {
-                    if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
+            }
+            Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
+                let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
+                if lhs.delimiter != rhs.delimiter {
+                    bail!("mismatched delimiter")
                 }
-                crate::Leaf::Ident(ident) => {
-                    if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
+                let mut src = TtIter::new(rhs);
+                match_subtree(bindings, lhs, &mut src)?;
+                if src.len() > 0 {
+                    bail!("leftover tokens");
                 }
-                crate::Leaf::Literal(literal) => {
-                    if input.eat_literal().map(|i| &i.text) != Some(&literal.text) {
-                        return Err(ExpandError::UnexpectedToken);
+            }
+            Op::Var { name, kind } => {
+                let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
+                match match_meta_var(kind.as_str(), src)? {
+                    Some(fragment) => {
+                        bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
                     }
+                    None => bindings.push_optional(name),
                 }
+                ()
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                match_repeat(bindings, subtree, kind, separator, src)?
+            }
+        }
+    }
+    Ok(())
+}
+
+impl<'a> TtIter<'a> {
+    fn eat_separator(&mut self, separator: &Separator) -> bool {
+        let mut fork = self.clone();
+        let ok = match separator {
+            Separator::Ident(lhs) => match fork.expect_ident() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
             },
-            crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
-                // Dirty hack to make macro-expansion terminate.
-                // This should be replaced by a propper macro-by-example implementation
-                let mut limit = 65536;
-                let mut counter = 0;
-
-                let mut memento = input.save();
-
-                loop {
-                    match match_lhs(subtree, input) {
-                        Ok(nested) => {
-                            limit -= 1;
-                            if limit == 0 {
-                                log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
-                                break;
-                            }
-
-                            memento = input.save();
-                            res.push_nested(counter, nested)?;
-                            counter += 1;
-                            if counter == 1 {
-                                if let crate::RepeatKind::ZeroOrOne = kind {
-                                    break;
-                                }
-                            }
-
-                            if let Some(separator) = separator {
-                                if !input
-                                    .eat_seperator()
-                                    .map(|sep| sep == *separator)
-                                    .unwrap_or(false)
-                                {
-                                    input.rollback(memento);
-                                    break;
-                                }
-                            }
-                        }
-                        Err(_) => {
-                            input.rollback(memento);
-                            break;
-                        }
-                    }
-                }
+            Separator::Literal(lhs) => match fork.expect_literal() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
+            },
+            Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
+                Ok(rhs) => rhs.char == lhs.char,
+                _ => false,
+            }),
+        };
+        if ok {
+            *self = fork;
+        }
+        ok
+    }
 
-                match kind {
-                    crate::RepeatKind::OneOrMore if counter == 0 => {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-                    _ if counter == 0 => {
-                        // Collect all empty variables in subtrees
-                        collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
-                    }
-                    _ => {}
+    pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
+        let ident = self.expect_ident()?;
+        // check if it start from "`"
+        if ident.text.chars().next() != Some('\'') {
+            return Err(());
+        }
+        Ok(ident)
+    }
+
+    pub(crate) fn expect_fragment(
+        &mut self,
+        fragment_kind: ra_parser::FragmentKind,
+    ) -> Result<tt::TokenTree, ()> {
+        pub(crate) struct OffsetTokenSink<'a> {
+            pub(crate) cursor: Cursor<'a>,
+            pub(crate) error: bool,
+        }
+
+        impl<'a> TreeSink for OffsetTokenSink<'a> {
+            fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
+                for _ in 0..n_tokens {
+                    self.cursor = self.cursor.bump_subtree();
                 }
             }
-            crate::TokenTree::Subtree(subtree) => {
-                let input_subtree =
-                    input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
-                if subtree.delimiter != input_subtree.delimiter {
-                    return Err(ExpandError::UnexpectedToken);
+            fn start_node(&mut self, _kind: SyntaxKind) {}
+            fn finish_node(&mut self) {}
+            fn error(&mut self, _error: ra_parser::ParseError) {
+                self.error = true;
+            }
+        }
+
+        let buffer = TokenBuffer::new(self.inner.as_slice());
+        let mut src = SubtreeTokenSource::new(&buffer);
+        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
+
+        ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind);
+
+        if !sink.cursor.is_root() || sink.error {
+            return Err(());
+        }
+
+        let mut curr = buffer.begin();
+        let mut res = vec![];
+
+        while curr != sink.cursor {
+            if let Some(token) = curr.token_tree() {
+                res.push(token);
+            }
+            curr = curr.bump();
+        }
+        self.inner = self.inner.as_slice()[res.len()..].iter();
+        match res.len() {
+            0 => Err(()),
+            1 => Ok(res[0].clone()),
+            _ => Ok(tt::TokenTree::Subtree(tt::Subtree {
+                delimiter: tt::Delimiter::None,
+                token_trees: res.into_iter().cloned().collect(),
+            })),
+        }
+    }
+
+    pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
+        let mut fork = self.clone();
+        match fork.expect_fragment(Visibility) {
+            Ok(tt) => {
+                *self = fork;
+                Some(tt)
+            }
+            Err(()) => None,
+        }
+    }
+}
+
+pub(super) fn match_repeat(
+    bindings: &mut Bindings,
+    pattern: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a propper macro-by-example implementation
+    let mut limit = 65536;
+    let mut counter = 0;
+
+    for i in 0.. {
+        let mut fork = src.clone();
+
+        if let Some(separator) = &separator {
+            if i != 0 && !fork.eat_separator(separator) {
+                break;
+            }
+        }
+
+        let mut nested = Bindings::default();
+        match match_subtree(&mut nested, pattern, &mut fork) {
+            Ok(()) => {
+                limit -= 1;
+                if limit == 0 {
+                    log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator);
+                    break;
                 }
+                *src = fork;
 
-                let mut input = TtCursor::new(input_subtree);
-                let bindings = match_lhs(&subtree, &mut input)?;
-                if !input.is_eof() {
-                    return Err(ExpandError::UnexpectedToken);
+                bindings.push_nested(counter, nested)?;
+                counter += 1;
+                if counter == 1 {
+                    if let RepeatKind::ZeroOrOne = kind {
+                        break;
+                    }
                 }
+            }
+            Err(_) => break,
+        }
+    }
 
-                res.merge(bindings);
+    match (kind, counter) {
+        (RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken),
+        (_, 0) => {
+            // Collect all empty variables in subtrees
+            let mut vars = Vec::new();
+            collect_vars(&mut vars, pattern)?;
+            for var in vars {
+                bindings.push_empty(&var)
             }
         }
+        _ => (),
     }
-    Ok(res)
+    Ok(())
 }
 
-fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> {
+fn match_meta_var(kind: &str, input: &mut TtIter) -> Result<Option<Fragment>, ExpandError> {
     let fragment = match kind {
         "path" => Path,
         "expr" => Expr,
@@ -169,17 +298,20 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
         _ => {
             let tt = match kind {
                 "ident" => {
-                    let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
+                    let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone();
                     tt::Leaf::from(ident).into()
                 }
-                "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(),
-                "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(),
+                "tt" => input.next().ok_or_else(|| err!())?.clone(),
+                "lifetime" => {
+                    let ident = input.expect_lifetime().map_err(|()| err!())?;
+                    tt::Leaf::Ident(ident.clone()).into()
+                }
                 "literal" => {
-                    let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
+                    let literal = input.expect_literal().map_err(|()| err!())?.clone();
                     tt::Leaf::from(literal).into()
                 }
                 // `vis` is optional
-                "vis" => match input.try_eat_vis() {
+                "vis" => match input.eat_vis() {
                     Some(vis) => vis,
                     None => return Ok(None),
                 },
@@ -188,28 +320,19 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
             return Ok(Some(Fragment::Tokens(tt)));
         }
     };
-    let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?;
+    let tt = input.expect_fragment(fragment).map_err(|()| err!())?;
     let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
     Ok(Some(fragment))
 }
 
-fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
-    let mut res = Vec::new();
-
-    for tkn in subtree.token_trees.iter() {
-        match tkn {
-            crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
-                res.push(text.clone());
-            }
-            crate::TokenTree::Subtree(subtree) => {
-                res.extend(collect_vars(subtree));
-            }
-            crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
-                res.extend(collect_vars(subtree));
-            }
-            _ => {}
+fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::Var { name, .. } => buf.push(name.clone()),
+            Op::TokenTree(tt::TokenTree::Leaf(_)) => (),
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?,
+            Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
         }
     }
-
-    res
+    Ok(())
 }
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs
index a3df1b7de..c22680b93 100644
--- a/crates/ra_mbe/src/mbe_expander/transcriber.rs
+++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs
@@ -1,16 +1,20 @@
+//! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like
+//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
+
 use ra_syntax::SmolStr;
 
 use crate::{
     mbe_expander::{Binding, Bindings, Fragment},
+    parser::{parse_template, Op, RepeatKind, Separator},
     ExpandError,
 };
 
 impl Bindings {
-    fn contains(&self, name: &SmolStr) -> bool {
+    fn contains(&self, name: &str) -> bool {
         self.inner.contains_key(name)
     }
 
-    fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
+    fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
         let mut b = self.inner.get(name).ok_or_else(|| {
             ExpandError::BindingError(format!("could not find binding `{}`", name))
         })?;
@@ -43,11 +47,12 @@ impl Bindings {
 }
 
 pub(super) fn transcribe(
+    template: &tt::Subtree,
     bindings: &Bindings,
-    template: &crate::Subtree,
 ) -> Result<tt::Subtree, ExpandError> {
+    assert!(template.delimiter == tt::Delimiter::None);
     let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
-    expand_subtree(template, &mut ctx)
+    expand_subtree(&mut ctx, template)
 }
 
 #[derive(Debug)]
@@ -57,159 +62,158 @@ struct ExpandCtx<'a> {
     var_expanded: bool,
 }
 
-fn expand_subtree(
-    template: &crate::Subtree,
-    ctx: &mut ExpandCtx,
-) -> Result<tt::Subtree, ExpandError> {
+fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
     let mut buf: Vec<tt::TokenTree> = Vec::new();
-    for tt in template.token_trees.iter() {
-        let tt = expand_tt(tt, ctx)?;
-        push_fragment(&mut buf, tt);
+    for op in parse_template(template) {
+        match op? {
+            Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()),
+            Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
+                let tt = expand_subtree(ctx, tt)?;
+                buf.push(tt.into());
+            }
+            Op::Var { name, kind: _ } => {
+                let fragment = expand_var(ctx, name)?;
+                push_fragment(&mut buf, fragment);
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                let fragment = expand_repeat(ctx, subtree, kind, separator)?;
+                push_fragment(&mut buf, fragment)
+            }
+        }
     }
-
     Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
 }
 
-fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> {
-    let res: tt::TokenTree = match template {
-        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
-        crate::TokenTree::Repeat(repeat) => {
-            let mut buf: Vec<tt::TokenTree> = Vec::new();
-            ctx.nesting.push(0);
-            // Dirty hack to make macro-expansion terminate.
-            // This should be replaced by a propper macro-by-example implementation
-            let mut limit = 65536;
-            let mut has_seps = 0;
-            let mut counter = 0;
-
-            // We store the old var expanded value, and restore it later
-            // It is because before this `$repeat`,
-            // it is possible some variables already expanad in the same subtree
-            //
-            // `some_var_expanded` keep check if the deeper subtree has expanded variables
-            let mut some_var_expanded = false;
-            let old_var_expanded = ctx.var_expanded;
-            ctx.var_expanded = false;
-
-            while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
-                // if no var expanded in the child, we count it as a fail
-                if !ctx.var_expanded {
-                    break;
-                }
+fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> {
+    let res = if v == "crate" {
+        // FIXME: Properly handle $crate token
+        let tt =
+            tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
+                .into();
+        Fragment::Tokens(tt)
+    } else if !ctx.bindings.contains(v) {
+        // Note that it is possible to have a `$var` inside a macro which is not bound.
+        // For example:
+        // ```
+        // macro_rules! foo {
+        //     ($a:ident, $b:ident, $c:tt) => {
+        //         macro_rules! bar {
+        //             ($bi:ident) => {
+        //                 fn $bi() -> u8 {$c}
+        //             }
+        //         }
+        //     }
+        // ```
+        // We just treat it a normal tokens
+        let tt = tt::Subtree {
+            delimiter: tt::Delimiter::None,
+            token_trees: vec![
+                tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(),
+                tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
+                    .into(),
+            ],
+        }
+        .into();
+        Fragment::Tokens(tt)
+    } else {
+        let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone();
+        ctx.var_expanded = true;
+        fragment
+    };
+    Ok(res)
+}
 
-                // Reset `ctx.var_expandeded` to see if there is other expanded variable
-                // in the next matching
-                some_var_expanded = true;
-                ctx.var_expanded = false;
-
-                counter += 1;
-                limit -= 1;
-                if limit == 0 {
-                    log::warn!(
-                        "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
-                        template,
-                        ctx
-                    );
-                    break;
-                }
+fn expand_repeat(
+    ctx: &mut ExpandCtx,
+    template: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+) -> Result<Fragment, ExpandError> {
+    let mut buf: Vec<tt::TokenTree> = Vec::new();
+    ctx.nesting.push(0);
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a propper macro-by-example implementation
+    let mut limit = 65536;
+    let mut has_seps = 0;
+    let mut counter = 0;
+
+    // We store the old var expanded value, and restore it later
+    // It is because before this `$repeat`,
+    // it is possible some variables already expanad in the same subtree
+    //
+    // `some_var_expanded` keep check if the deeper subtree has expanded variables
+    let mut some_var_expanded = false;
+    let old_var_expanded = ctx.var_expanded;
+    ctx.var_expanded = false;
+
+    while let Ok(mut t) = expand_subtree(ctx, template) {
+        t.delimiter = tt::Delimiter::None;
+        // if no var expanded in the child, we count it as a fail
+        if !ctx.var_expanded {
+            break;
+        }
 
-                let idx = ctx.nesting.pop().unwrap();
-                ctx.nesting.push(idx + 1);
-                push_subtree(&mut buf, t);
-
-                if let Some(ref sep) = repeat.separator {
-                    match sep {
-                        crate::Separator::Ident(ident) => {
-                            has_seps = 1;
-                            buf.push(tt::Leaf::from(ident.clone()).into());
-                        }
-                        crate::Separator::Literal(lit) => {
-                            has_seps = 1;
-                            buf.push(tt::Leaf::from(lit.clone()).into());
-                        }
-
-                        crate::Separator::Puncts(puncts) => {
-                            has_seps = puncts.len();
-                            for punct in puncts {
-                                buf.push(tt::Leaf::from(*punct).into());
-                            }
-                        }
-                    }
+        // Reset `ctx.var_expandeded` to see if there is other expanded variable
+        // in the next matching
+        some_var_expanded = true;
+        ctx.var_expanded = false;
+
+        counter += 1;
+        limit -= 1;
+        if limit == 0 {
+            log::warn!(
+                "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
+                template,
+                ctx
+            );
+            break;
+        }
+
+        let idx = ctx.nesting.pop().unwrap();
+        ctx.nesting.push(idx + 1);
+        push_subtree(&mut buf, t);
+
+        if let Some(ref sep) = separator {
+            match sep {
+                Separator::Ident(ident) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(ident.clone()).into());
+                }
+                Separator::Literal(lit) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(lit.clone()).into());
                 }
 
-                if let crate::RepeatKind::ZeroOrOne = repeat.kind {
-                    break;
+                Separator::Puncts(puncts) => {
+                    has_seps = puncts.len();
+                    for punct in puncts {
+                        buf.push(tt::Leaf::from(*punct).into());
+                    }
                 }
             }
+        }
 
-            // Restore the `var_expanded` by combining old one and the new one
-            ctx.var_expanded = some_var_expanded || old_var_expanded;
+        if RepeatKind::ZeroOrOne == kind {
+            break;
+        }
+    }
 
-            ctx.nesting.pop().unwrap();
-            for _ in 0..has_seps {
-                buf.pop();
-            }
+    // Restore the `var_expanded` by combining old one and the new one
+    ctx.var_expanded = some_var_expanded || old_var_expanded;
 
-            if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
-                return Err(ExpandError::UnexpectedToken);
-            }
+    ctx.nesting.pop().unwrap();
+    for _ in 0..has_seps {
+        buf.pop();
+    }
 
-            // Check if it is a single token subtree without any delimiter
-            // e.g {Delimiter:None> ['>'] /Delimiter:None>}
-            tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into()
-        }
-        crate::TokenTree::Leaf(leaf) => match leaf {
-            crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident {
-                text: ident.text.clone(),
-                id: tt::TokenId::unspecified(),
-            })
-            .into(),
-            crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
-            crate::Leaf::Var(v) => {
-                if v.text == "crate" {
-                    // FIXME: Properly handle $crate token
-                    tt::Leaf::from(tt::Ident {
-                        text: "$crate".into(),
-                        id: tt::TokenId::unspecified(),
-                    })
-                    .into()
-                } else if !ctx.bindings.contains(&v.text) {
-                    // Note that it is possible to have a `$var` inside a macro which is not bound.
-                    // For example:
-                    // ```
-                    // macro_rules! foo {
-                    //     ($a:ident, $b:ident, $c:tt) => {
-                    //         macro_rules! bar {
-                    //             ($bi:ident) => {
-                    //                 fn $bi() -> u8 {$c}
-                    //             }
-                    //         }
-                    //     }
-                    // ```
-                    // We just treat it a normal tokens
-                    tt::Subtree {
-                        delimiter: tt::Delimiter::None,
-                        token_trees: vec![
-                            tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
-                                .into(),
-                            tt::Leaf::from(tt::Ident {
-                                text: v.text.clone(),
-                                id: tt::TokenId::unspecified(),
-                            })
-                            .into(),
-                        ],
-                    }
-                    .into()
-                } else {
-                    let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
-                    ctx.var_expanded = true;
-                    return Ok(fragment);
-                }
-            }
-            crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
-        },
-    };
-    Ok(Fragment::Tokens(res))
+    if RepeatKind::OneOrMore == kind && counter == 0 {
+        return Err(ExpandError::UnexpectedToken);
+    }
+
+    // Check if it is a single token subtree without any delimiter
+    // e.g {Delimiter:None> ['>'] /Delimiter:None>}
+    let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into();
+    Ok(Fragment::Tokens(tt))
 }
 
 fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {