Fix incorrect repeat sep eating

3 files changed, 304 insertions, 24 deletions

diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index eedc0c5dd..d7b18dd0f 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -384,7 +384,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
 "#,
         );
 
-        assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}");
+        assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar ()}");
     }
 
     #[test]
@@ -1066,4 +1066,260 @@ macro_rules! int_base {
         "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt :: Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
         );
     }
+
+    #[test]
+    fn test_generate_pattern_iterators() {
+        // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs
+        let rules = create_rules(
+            r#"
+macro_rules! generate_pattern_iterators {        
+        { double ended; with $(#[$common_stability_attribute:meta])*,
+                           $forward_iterator:ident,
+                           $reverse_iterator:ident, $iterty:ty
+        } => {
+            fn foo(){}
+        }
+}
+"#,
+        );
+
+        assert_expansion(&rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str )"#, 
+        "fn foo () {}");
+    }
+
+    #[test]
+    fn test_impl_fn_for_zst() {
+        // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs
+        let rules = create_rules(
+            r#"
+macro_rules! impl_fn_for_zst  {        
+        {  $( $( #[$attr: meta] )*
+        struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
+            |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty
+$body: block; )+
+        } => {
+            fn foo(){}
+        }
+}
+"#,
+        );
+
+        assert_expansion(&rules, r#"
+impl_fn_for_zst !   { 
+     # [ derive ( Clone ) ] 
+     struct   CharEscapeDebugContinue   impl   Fn   =   | c :   char |   ->   char :: EscapeDebug   { 
+         c . escape_debug_ext ( false ) 
+     } ; 
+
+     # [ derive ( Clone ) ] 
+     struct   CharEscapeUnicode   impl   Fn   =   | c :   char |   ->   char :: EscapeUnicode   { 
+         c . escape_unicode ( ) 
+     } ; 
+     # [ derive ( Clone ) ] 
+     struct   CharEscapeDefault   impl   Fn   =   | c :   char |   ->   char :: EscapeDefault   { 
+         c . escape_default ( ) 
+     } ; 
+ }
+"#, 
+        "fn foo () {}");
+    }
+
+    #[test]
+    fn test_impl_nonzero_fmt() {
+        // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
+        let rules = create_rules(
+            r#"
+        macro_rules! impl_nonzero_fmt {
+            ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
+                fn foo() {}
+            }
+        }
+"#,
+        );
+
+        assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#, 
+        "fn foo () {}");
+    }
+
+    #[test]
+    fn test_tuple_impls() {
+        // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
+        let rules = create_rules(
+            r#"
+    macro_rules! tuple_impls {
+    ($(
+        $Tuple:ident {
+            $(($idx:tt) -> $T:ident)+
+        }
+    )+) => {
+        $(
+            #[stable(feature = "rust1", since = "1.0.0")]
+            impl<$($T:PartialEq),+> PartialEq for ($($T,)+) where last_type!($($T,)+): ?Sized {
+                #[inline]
+                fn eq(&self, other: &($($T,)+)) -> bool {
+                    $(self.$idx == other.$idx)&&+
+                }
+                #[inline]
+                fn ne(&self, other: &($($T,)+)) -> bool {
+                    $(self.$idx != other.$idx)||+
+                }
+            }
+
+            #[stable(feature = "rust1", since = "1.0.0")]
+            impl<$($T:Eq),+> Eq for ($($T,)+) where last_type!($($T,)+): ?Sized {}
+
+            #[stable(feature = "rust1", since = "1.0.0")]
+            impl<$($T:PartialOrd + PartialEq),+> PartialOrd for ($($T,)+)
+                    where last_type!($($T,)+): ?Sized {
+                #[inline]
+                fn partial_cmp(&self, other: &($($T,)+)) -> Option<Ordering> {
+                    lexical_partial_cmp!($(self.$idx, other.$idx),+)
+                }
+                #[inline]
+                fn lt(&self, other: &($($T,)+)) -> bool {
+                    lexical_ord!(lt, $(self.$idx, other.$idx),+)
+                }
+                #[inline]
+                fn le(&self, other: &($($T,)+)) -> bool {
+                    lexical_ord!(le, $(self.$idx, other.$idx),+)
+                }
+                #[inline]
+                fn ge(&self, other: &($($T,)+)) -> bool {
+                    lexical_ord!(ge, $(self.$idx, other.$idx),+)
+                }
+                #[inline]
+                fn gt(&self, other: &($($T,)+)) -> bool {
+                    lexical_ord!(gt, $(self.$idx, other.$idx),+)
+                }
+            }
+
+            #[stable(feature = "rust1", since = "1.0.0")]
+            impl<$($T:Ord),+> Ord for ($($T,)+) where last_type!($($T,)+): ?Sized {
+                #[inline]
+                fn cmp(&self, other: &($($T,)+)) -> Ordering {
+                    lexical_cmp!($(self.$idx, other.$idx),+)
+                }
+            }
+
+            #[stable(feature = "rust1", since = "1.0.0")]
+            impl<$($T:Default),+> Default for ($($T,)+) {
+                #[inline]
+                fn default() -> ($($T,)+) {
+                    ($({ let x: $T = Default::default(); x},)+)
+                }
+            }
+        )+
+    }
+}"#,
+        );
+
+        assert_expansion(
+            &rules,
+            r#"tuple_impls ! { 
+    Tuple1   { 
+         ( 0 )   ->   A 
+     } 
+     Tuple2   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+     } 
+     Tuple3   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+     } 
+     Tuple4   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+     } 
+     Tuple5   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+     } 
+     Tuple6   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+     } 
+     Tuple7   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+         ( 6 )   ->   G 
+     } 
+     Tuple8   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+         ( 6 )   ->   G 
+         ( 7 )   ->   H 
+     } 
+     Tuple9   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+         ( 6 )   ->   G 
+         ( 7 )   ->   H 
+         ( 8 )   ->   I 
+     } 
+     Tuple10   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+         ( 6 )   ->   G 
+         ( 7 )   ->   H 
+         ( 8 )   ->   I 
+         ( 9 )   ->   J 
+     } 
+     Tuple11   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+         ( 6 )   ->   G 
+         ( 7 )   ->   H 
+         ( 8 )   ->   I 
+         ( 9 )   ->   J 
+         ( 10 )   ->   K 
+     } 
+     Tuple12   { 
+         ( 0 )   ->   A 
+         ( 1 )   ->   B 
+         ( 2 )   ->   C 
+         ( 3 )   ->   D 
+         ( 4 )   ->   E 
+         ( 5 )   ->   F 
+         ( 6 )   ->   G 
+         ( 7 )   ->   H 
+         ( 8 )   ->   I 
+         ( 9 )   ->   J 
+         ( 10 )   ->   K 
+         ( 11 )   ->   L 
+     } 
+}"#,
+            "fn foo () {}",
+        );
+    }
 }
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 00fb09a3b..91b6db522 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -179,10 +179,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                         // Enable followiing code when everything is fixed
                         // At least we can dogfood itself to not stackoverflow
                         //
-                        // "tt" => {
-                        //     let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
-                        //     res.inner.insert(text.clone(), Binding::Simple(token.into()));
-                        // }
+                        "tt" => {
+                            let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
+                            res.inner.insert(text.clone(), Binding::Simple(token.into()));
+                        }
                         "item" => {
                             let item =
                                 input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone();
@@ -226,18 +226,36 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                 // This should be replaced by a propper macro-by-example implementation
                 let mut limit = 128;
                 let mut counter = 0;
-                while let Ok(nested) = match_lhs(subtree, input) {
-                    counter += 1;
-                    limit -= 1;
-                    if limit == 0 {
-                        break;
-                    }
-                    res.push_nested(nested)?;
-                    if let Some(separator) = *separator {
-                        if !input.is_eof() {
-                            if input.eat_punct().map(|p| p.char) != Some(separator) {
-                                return Err(ExpandError::UnexpectedToken);
+
+                let mut memento = input.save();
+
+                loop {
+                    match match_lhs(subtree, input) {
+                        Ok(nested) => {
+                            counter += 1;
+                            limit -= 1;
+                            if limit == 0 {
+                                break;
                             }
+
+                            memento = input.save();
+                            res.push_nested(nested)?;
+                            if counter == 1 {
+                                if let crate::RepeatKind::ZeroOrOne = kind {
+                                    break;
+                                }
+                            }
+
+                            if let Some(separator) = *separator {
+                                if input.eat_punct().map(|p| p.char) != Some(separator) {
+                                    input.rollback(memento);
+                                    break;
+                                }
+                            }
+                        }
+                        Err(_) => {
+                            input.rollback(memento);
+                            break;
                         }
                     }
                 }
@@ -246,10 +264,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                     crate::RepeatKind::OneOrMore if counter == 0 => {
                         return Err(ExpandError::UnexpectedToken);
                     }
-                    crate::RepeatKind::ZeroOrOne if counter > 1 => {
-                        return Err(ExpandError::UnexpectedToken);
-                    }
-
                     _ => {}
                 }
             }
@@ -333,10 +347,7 @@ fn expand_tt(
                 }
             }
             nesting.pop().unwrap();
-
-            // Dirty hack for remove the last sep
-            // if it is a "," undo the push
-            if has_sep && repeat.separator.unwrap() == ',' {
+            if has_sep {
                 token_trees.pop();
             }
 
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs
index 741b5ea1c..87bcf8b0d 100644
--- a/crates/ra_mbe/src/tt_cursor.rs
+++ b/crates/ra_mbe/src/tt_cursor.rs
@@ -7,6 +7,10 @@ pub(crate) struct TtCursor<'a> {
     pos: usize,
 }
 
+pub(crate) struct TtCursorMemento {
+    pos: usize,
+}
+
 impl<'a> TtCursor<'a> {
     pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> {
         TtCursor { subtree, pos: 0 }
@@ -157,4 +161,13 @@ impl<'a> TtCursor<'a> {
             Err(ParseError::Expected(format!("`{}`", char)))
         }
     }
+
+    #[must_use]
+    pub(crate) fn save(&self) -> TtCursorMemento {
+        TtCursorMemento { pos: self.pos }
+    }
+
+    pub(crate) fn rollback(&mut self, memento: TtCursorMemento) {
+        self.pos = memento.pos;
+    }
 }