1 files changed, 123 insertions, 44 deletions

diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 00fb09a3b..d5189b537 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -21,7 +21,10 @@ fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, E
     if !input.is_eof() {
         return Err(ExpandError::UnexpectedToken);
     }
-    expand_subtree(&rule.rhs, &bindings, &mut Vec::new())
+
+    let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
+
+    expand_subtree(&rule.rhs, &mut ctx)
 }
 
 /// The actual algorithm for expansion is not too hard, but is pretty tricky.
@@ -179,10 +182,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                         // Enable followiing code when everything is fixed
                         // At least we can dogfood itself to not stackoverflow
                         //
-                        // "tt" => {
-                        //     let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
-                        //     res.inner.insert(text.clone(), Binding::Simple(token.into()));
-                        // }
+                        "tt" => {
+                            let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
+                            res.inner.insert(text.clone(), Binding::Simple(token.into()));
+                        }
                         "item" => {
                             let item =
                                 input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone();
@@ -196,6 +199,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                         "literal" => {
                             let literal =
                                 input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
+
                             res.inner.insert(
                                 text.clone(),
                                 Binding::Simple(tt::Leaf::from(literal).into()),
@@ -210,7 +214,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                     }
                 }
                 crate::Leaf::Punct(punct) => {
-                    if input.eat_punct() != Some(punct) {
+                    if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
                         return Err(ExpandError::UnexpectedToken);
                     }
                 }
@@ -224,20 +228,54 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
             crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
                 // Dirty hack to make macro-expansion terminate.
                 // This should be replaced by a propper macro-by-example implementation
-                let mut limit = 128;
+                let mut limit = 65536;
                 let mut counter = 0;
-                while let Ok(nested) = match_lhs(subtree, input) {
-                    counter += 1;
-                    limit -= 1;
-                    if limit == 0 {
-                        break;
-                    }
-                    res.push_nested(nested)?;
-                    if let Some(separator) = *separator {
-                        if !input.is_eof() {
-                            if input.eat_punct().map(|p| p.char) != Some(separator) {
-                                return Err(ExpandError::UnexpectedToken);
+
+                let mut memento = input.save();
+
+                loop {
+                    match match_lhs(subtree, input) {
+                        Ok(nested) => {
+                            counter += 1;
+                            limit -= 1;
+                            if limit == 0 {
+                                log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
+                                break;
+                            }
+
+                            memento = input.save();
+                            res.push_nested(nested)?;
+                            if counter == 1 {
+                                if let crate::RepeatKind::ZeroOrOne = kind {
+                                    break;
+                                }
                             }
+
+                            if let Some(separator) = separator {
+                                use crate::Separator::*;
+
+                                if !input
+                                    .eat_seperator()
+                                    .map(|sep| match (sep, separator) {
+                                        (Ident(ref a), Ident(ref b)) => a.text == b.text,
+                                        (Literal(ref a), Literal(ref b)) => a.text == b.text,
+                                        (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
+                                            let a_iter = a.iter().map(|a| a.char);
+                                            let b_iter = b.iter().map(|b| b.char);
+                                            a_iter.eq(b_iter)
+                                        }
+                                        _ => false,
+                                    })
+                                    .unwrap_or(false)
+                                {
+                                    input.rollback(memento);
+                                    break;
+                                }
+                            }
+                        }
+                        Err(_) => {
+                            input.rollback(memento);
+                            break;
                         }
                     }
                 }
@@ -246,10 +284,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                     crate::RepeatKind::OneOrMore if counter == 0 => {
                         return Err(ExpandError::UnexpectedToken);
                     }
-                    crate::RepeatKind::ZeroOrOne if counter > 1 => {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-
                     _ => {}
                 }
             }
@@ -273,15 +307,21 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
     Ok(res)
 }
 
+#[derive(Debug)]
+struct ExpandCtx<'a> {
+    bindings: &'a Bindings,
+    nesting: Vec<usize>,
+    var_expanded: bool,
+}
+
 fn expand_subtree(
     template: &crate::Subtree,
-    bindings: &Bindings,
-    nesting: &mut Vec<usize>,
+    ctx: &mut ExpandCtx,
 ) -> Result<tt::Subtree, ExpandError> {
     let token_trees = template
         .token_trees
         .iter()
-        .map(|it| expand_tt(it, bindings, nesting))
+        .map(|it| expand_tt(it, ctx))
         .collect::<Result<Vec<_>, ExpandError>>()?;
 
     Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
@@ -303,43 +343,81 @@ fn reduce_single_token(mut subtree: tt::Subtree) -> tt::TokenTree {
 
 fn expand_tt(
     template: &crate::TokenTree,
-    bindings: &Bindings,
-    nesting: &mut Vec<usize>,
+    ctx: &mut ExpandCtx,
 ) -> Result<tt::TokenTree, ExpandError> {
     let res: tt::TokenTree = match template {
-        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(),
+        crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
         crate::TokenTree::Repeat(repeat) => {
             let mut token_trees: Vec<tt::TokenTree> = Vec::new();
-            nesting.push(0);
+            ctx.nesting.push(0);
             // Dirty hack to make macro-expansion terminate.
             // This should be replaced by a propper macro-by-example implementation
-            let mut limit = 128;
-            let mut has_sep = false;
+            let mut limit = 65536;
+            let mut has_seps = 0;
+            let mut counter = 0;
 
-            while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) {
+            let mut some_var_expanded = false;
+            ctx.var_expanded = false;
+
+            while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
+                // if no var expaned in the child, we count it as a fail
+                if !ctx.var_expanded {
+                    break;
+                }
+                some_var_expanded = true;
+                ctx.var_expanded = false;
+
+                counter += 1;
                 limit -= 1;
                 if limit == 0 {
+                    log::warn!(
+                        "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
+                        template,
+                        ctx
+                    );
                     break;
                 }
-                let idx = nesting.pop().unwrap();
-                nesting.push(idx + 1);
+
+                let idx = ctx.nesting.pop().unwrap();
+                ctx.nesting.push(idx + 1);
                 token_trees.push(reduce_single_token(t).into());
 
-                if let Some(sep) = repeat.separator {
-                    let punct =
-                        tt::Leaf::from(tt::Punct { char: sep, spacing: tt::Spacing::Alone });
-                    token_trees.push(punct.into());
-                    has_sep = true;
+                if let Some(ref sep) = repeat.separator {
+                    match sep {
+                        crate::Separator::Ident(ident) => {
+                            has_seps = 1;
+                            token_trees.push(tt::Leaf::from(ident.clone()).into());
+                        }
+                        crate::Separator::Literal(lit) => {
+                            has_seps = 1;
+                            token_trees.push(tt::Leaf::from(lit.clone()).into());
+                        }
+
+                        crate::Separator::Puncts(puncts) => {
+                            has_seps = puncts.len();
+                            for punct in puncts {
+                                token_trees.push(tt::Leaf::from(*punct).into());
+                            }
+                        }
+                    }
+                }
+
+                if let crate::RepeatKind::ZeroOrOne = repeat.kind {
+                    break;
                 }
             }
-            nesting.pop().unwrap();
 
-            // Dirty hack for remove the last sep
-            // if it is a "," undo the push
-            if has_sep && repeat.separator.unwrap() == ',' {
+            ctx.var_expanded = some_var_expanded;
+
+            ctx.nesting.pop().unwrap();
+            for _ in 0..has_seps {
                 token_trees.pop();
             }
 
+            if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
+                return Err(ExpandError::UnexpectedToken);
+            }
+
             // Check if it is a singel token subtree without any delimiter
             // e.g {Delimiter:None> ['>'] /Delimiter:None>}
             reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None })
@@ -356,7 +434,8 @@ fn expand_tt(
                     tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
                         .into()
                 } else {
-                    let tkn = bindings.get(&v.text, nesting)?.clone();
+                    let tkn = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
+                    ctx.var_expanded = true;
 
                     if let tt::TokenTree::Subtree(subtree) = tkn {
                         reduce_single_token(subtree)