From e4e2338f97d734624fb21b7972c73288ca04e65e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Wed, 24 Apr 2019 02:59:38 +0800 Subject: Fix incorrect repeat sep eating --- crates/ra_mbe/src/lib.rs | 258 +++++++++++++++++++++++++++++++++++++- crates/ra_mbe/src/mbe_expander.rs | 57 +++++---- crates/ra_mbe/src/tt_cursor.rs | 13 ++ 3 files changed, 304 insertions(+), 24 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index eedc0c5dd..d7b18dd0f 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -384,7 +384,7 @@ impl_froms!(TokenTree: Leaf, Subtree); "#, ); - assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}"); + assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar ()}"); } #[test] @@ -1066,4 +1066,260 @@ macro_rules! int_base { "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt :: Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}" ); } + + #[test] + fn test_generate_pattern_iterators() { + // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs + let rules = create_rules( + r#" +macro_rules! generate_pattern_iterators { + { double ended; with $(#[$common_stability_attribute:meta])*, + $forward_iterator:ident, + $reverse_iterator:ident, $iterty:ty + } => { + fn foo(){} + } +} +"#, + ); + + assert_expansion(&rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str )"#, + "fn foo () {}"); + } + + #[test] + fn test_impl_fn_for_zst() { + // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs + let rules = create_rules( + r#" +macro_rules! impl_fn_for_zst { + { $( $( #[$attr: meta] )* + struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn = + |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty +$body: block; )+ + } => { + fn foo(){} + } +} +"#, + ); + + assert_expansion(&rules, r#" +impl_fn_for_zst ! { + # [ derive ( Clone ) ] + struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug { + c . escape_debug_ext ( false ) + } ; + + # [ derive ( Clone ) ] + struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode { + c . escape_unicode ( ) + } ; + # [ derive ( Clone ) ] + struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault { + c . escape_default ( ) + } ; + } +"#, + "fn foo () {}"); + } + + #[test] + fn test_impl_nonzero_fmt() { + // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12 + let rules = create_rules( + r#" + macro_rules! impl_nonzero_fmt { + ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => { + fn foo() {} + } + } +"#, + ); + + assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#, + "fn foo () {}"); + } + + #[test] + fn test_tuple_impls() { + // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12 + let rules = create_rules( + r#" + macro_rules! tuple_impls { + ($( + $Tuple:ident { + $(($idx:tt) -> $T:ident)+ + } + )+) => { + $( + #[stable(feature = "rust1", since = "1.0.0")] + impl<$($T:PartialEq),+> PartialEq for ($($T,)+) where last_type!($($T,)+): ?Sized { + #[inline] + fn eq(&self, other: &($($T,)+)) -> bool { + $(self.$idx == other.$idx)&&+ + } + #[inline] + fn ne(&self, other: &($($T,)+)) -> bool { + $(self.$idx != other.$idx)||+ + } + } + + #[stable(feature = "rust1", since = "1.0.0")] + impl<$($T:Eq),+> Eq for ($($T,)+) where last_type!($($T,)+): ?Sized {} + + #[stable(feature = "rust1", since = "1.0.0")] + impl<$($T:PartialOrd + PartialEq),+> PartialOrd for ($($T,)+) + where last_type!($($T,)+): ?Sized { + #[inline] + fn partial_cmp(&self, other: &($($T,)+)) -> Option { + lexical_partial_cmp!($(self.$idx, other.$idx),+) + } + #[inline] + fn lt(&self, other: &($($T,)+)) -> bool { + lexical_ord!(lt, $(self.$idx, other.$idx),+) + } + #[inline] + fn le(&self, other: &($($T,)+)) -> bool { + lexical_ord!(le, $(self.$idx, other.$idx),+) + } + #[inline] + fn ge(&self, other: &($($T,)+)) -> bool { + lexical_ord!(ge, $(self.$idx, other.$idx),+) + } + #[inline] + fn gt(&self, other: &($($T,)+)) -> bool { + lexical_ord!(gt, $(self.$idx, other.$idx),+) + } + } + + #[stable(feature = "rust1", since = "1.0.0")] + impl<$($T:Ord),+> Ord for ($($T,)+) where last_type!($($T,)+): ?Sized { + #[inline] + fn cmp(&self, other: &($($T,)+)) -> Ordering { + lexical_cmp!($(self.$idx, other.$idx),+) + } + } + + #[stable(feature = "rust1", since = "1.0.0")] + impl<$($T:Default),+> Default for ($($T,)+) { + #[inline] + fn default() -> ($($T,)+) { + ($({ let x: $T = Default::default(); x},)+) + } + } + )+ + } +}"#, + ); + + assert_expansion( + &rules, + r#"tuple_impls ! { + Tuple1 { + ( 0 ) -> A + } + Tuple2 { + ( 0 ) -> A + ( 1 ) -> B + } + Tuple3 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + } + Tuple4 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + } + Tuple5 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + } + Tuple6 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + } + Tuple7 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + ( 6 ) -> G + } + Tuple8 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + ( 6 ) -> G + ( 7 ) -> H + } + Tuple9 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + ( 6 ) -> G + ( 7 ) -> H + ( 8 ) -> I + } + Tuple10 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + ( 6 ) -> G + ( 7 ) -> H + ( 8 ) -> I + ( 9 ) -> J + } + Tuple11 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + ( 6 ) -> G + ( 7 ) -> H + ( 8 ) -> I + ( 9 ) -> J + ( 10 ) -> K + } + Tuple12 { + ( 0 ) -> A + ( 1 ) -> B + ( 2 ) -> C + ( 3 ) -> D + ( 4 ) -> E + ( 5 ) -> F + ( 6 ) -> G + ( 7 ) -> H + ( 8 ) -> I + ( 9 ) -> J + ( 10 ) -> K + ( 11 ) -> L + } +}"#, + "fn foo () {}", + ); + } } diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 00fb09a3b..91b6db522 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -179,10 +179,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { - // let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(); - // res.inner.insert(text.clone(), Binding::Simple(token.into())); - // } + "tt" => { + let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(); + res.inner.insert(text.clone(), Binding::Simple(token.into())); + } "item" => { let item = input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone(); @@ -226,18 +226,36 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { + counter += 1; + limit -= 1; + if limit == 0 { + break; } + + memento = input.save(); + res.push_nested(nested)?; + if counter == 1 { + if let crate::RepeatKind::ZeroOrOne = kind { + break; + } + } + + if let Some(separator) = *separator { + if input.eat_punct().map(|p| p.char) != Some(separator) { + input.rollback(memento); + break; + } + } + } + Err(_) => { + input.rollback(memento); + break; } } } @@ -246,10 +264,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { return Err(ExpandError::UnexpectedToken); } - crate::RepeatKind::ZeroOrOne if counter > 1 => { - return Err(ExpandError::UnexpectedToken); - } - _ => {} } } @@ -333,10 +347,7 @@ fn expand_tt( } } nesting.pop().unwrap(); - - // Dirty hack for remove the last sep - // if it is a "," undo the push - if has_sep && repeat.separator.unwrap() == ',' { + if has_sep { token_trees.pop(); } diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index 741b5ea1c..87bcf8b0d 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs @@ -7,6 +7,10 @@ pub(crate) struct TtCursor<'a> { pos: usize, } +pub(crate) struct TtCursorMemento { + pos: usize, +} + impl<'a> TtCursor<'a> { pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> { TtCursor { subtree, pos: 0 } @@ -157,4 +161,13 @@ impl<'a> TtCursor<'a> { Err(ParseError::Expected(format!("`{}`", char))) } } + + #[must_use] + pub(crate) fn save(&self) -> TtCursorMemento { + TtCursorMemento { pos: self.pos } + } + + pub(crate) fn rollback(&mut self, memento: TtCursorMemento) { + self.pos = memento.pos; + } } -- cgit v1.2.3 From 299d97b6d98cec673ff056c188ac45a17febc7d4 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Wed, 24 Apr 2019 23:01:32 +0800 Subject: Add handling `token` seperator in mbe --- crates/ra_mbe/Cargo.toml | 1 + crates/ra_mbe/src/lib.rs | 216 +++++------------------------------- crates/ra_mbe/src/mbe_expander.rs | 49 ++++++-- crates/ra_mbe/src/mbe_parser.rs | 40 ++++--- crates/ra_mbe/src/subtree_source.rs | 22 ++-- crates/ra_mbe/src/syntax_bridge.rs | 25 ++++- crates/ra_mbe/src/tt_cursor.rs | 91 +++++++++++++++ 7 files changed, 218 insertions(+), 226 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml index 1d0c2a340..1e5ed6907 100644 --- a/crates/ra_mbe/Cargo.toml +++ b/crates/ra_mbe/Cargo.toml @@ -10,3 +10,4 @@ ra_parser = { path = "../ra_parser" } tt = { path = "../ra_tt", package = "ra_tt" } itertools = "0.8.0" rustc-hash = "1.0.0" +smallvec = "0.6.9" diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index d7b18dd0f..7ebba807c 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -24,6 +24,7 @@ mod subtree_source; mod subtree_parser; use ra_syntax::SmolStr; +use smallvec::SmallVec; pub use tt::{Delimiter, Punct}; @@ -98,11 +99,18 @@ pub(crate) struct Subtree { pub(crate) token_trees: Vec, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Separator { + Literal(tt::Literal), + Ident(tt::Ident), + Puncts(SmallVec<[tt::Punct; 3]>), +} + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct Repeat { pub(crate) subtree: Subtree, pub(crate) kind: RepeatKind, - pub(crate) separator: Option, + pub(crate) separator: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -175,8 +183,8 @@ impl_froms!(TokenTree: Leaf, Subtree); let expansion = rules.expand(&invocation_tt).unwrap(); assert_eq!( expansion.to_string(), - "impl From < Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree :: Leaf (it)}} \ - impl From < Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree :: Subtree (it)}}" + "impl From for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \ + impl From for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}" ) } @@ -384,7 +392,7 @@ impl_froms!(TokenTree: Leaf, Subtree); "#, ); - assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar ()}"); + assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ;bar ()}"); } #[test] @@ -416,6 +424,18 @@ impl_froms!(TokenTree: Leaf, Subtree); assert_expansion(&rules, "foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}"); } + #[test] + fn test_match_group_with_multichar_sep() { + let rules = create_rules( + r#" + macro_rules! foo { + (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} ); + }"#, + ); + + assert_expansion(&rules, "foo! (fn baz {true true} )", "fn baz () -> bool {true &&true}"); + } + #[test] fn test_expand_to_item_list() { let rules = create_rules( @@ -597,7 +617,7 @@ MACRO_ITEMS@[0; 40) assert_expansion( &rules, "foo! { bar::::baz:: }", - "fn foo () {let a = bar :: < u8 > :: baz :: < u8 > ;}", + "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}", ); } @@ -891,7 +911,7 @@ MACRO_ITEMS@[0; 40) } "#, ); - assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref < 'a > {s : & 'a str}"#); + assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#); } #[test] @@ -1063,7 +1083,7 @@ macro_rules! int_base { ); assert_expansion(&rules, r#" int_base!{Binary for isize as usize -> Binary}"#, - "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt :: Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}" + "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}" ); } @@ -1140,186 +1160,4 @@ impl_fn_for_zst ! { assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#, "fn foo () {}"); } - - #[test] - fn test_tuple_impls() { - // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12 - let rules = create_rules( - r#" - macro_rules! tuple_impls { - ($( - $Tuple:ident { - $(($idx:tt) -> $T:ident)+ - } - )+) => { - $( - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:PartialEq),+> PartialEq for ($($T,)+) where last_type!($($T,)+): ?Sized { - #[inline] - fn eq(&self, other: &($($T,)+)) -> bool { - $(self.$idx == other.$idx)&&+ - } - #[inline] - fn ne(&self, other: &($($T,)+)) -> bool { - $(self.$idx != other.$idx)||+ - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:Eq),+> Eq for ($($T,)+) where last_type!($($T,)+): ?Sized {} - - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:PartialOrd + PartialEq),+> PartialOrd for ($($T,)+) - where last_type!($($T,)+): ?Sized { - #[inline] - fn partial_cmp(&self, other: &($($T,)+)) -> Option { - lexical_partial_cmp!($(self.$idx, other.$idx),+) - } - #[inline] - fn lt(&self, other: &($($T,)+)) -> bool { - lexical_ord!(lt, $(self.$idx, other.$idx),+) - } - #[inline] - fn le(&self, other: &($($T,)+)) -> bool { - lexical_ord!(le, $(self.$idx, other.$idx),+) - } - #[inline] - fn ge(&self, other: &($($T,)+)) -> bool { - lexical_ord!(ge, $(self.$idx, other.$idx),+) - } - #[inline] - fn gt(&self, other: &($($T,)+)) -> bool { - lexical_ord!(gt, $(self.$idx, other.$idx),+) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:Ord),+> Ord for ($($T,)+) where last_type!($($T,)+): ?Sized { - #[inline] - fn cmp(&self, other: &($($T,)+)) -> Ordering { - lexical_cmp!($(self.$idx, other.$idx),+) - } - } - - #[stable(feature = "rust1", since = "1.0.0")] - impl<$($T:Default),+> Default for ($($T,)+) { - #[inline] - fn default() -> ($($T,)+) { - ($({ let x: $T = Default::default(); x},)+) - } - } - )+ - } -}"#, - ); - - assert_expansion( - &rules, - r#"tuple_impls ! { - Tuple1 { - ( 0 ) -> A - } - Tuple2 { - ( 0 ) -> A - ( 1 ) -> B - } - Tuple3 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - } - Tuple4 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - } - Tuple5 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - } - Tuple6 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - } - Tuple7 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - ( 6 ) -> G - } - Tuple8 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - ( 6 ) -> G - ( 7 ) -> H - } - Tuple9 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - ( 6 ) -> G - ( 7 ) -> H - ( 8 ) -> I - } - Tuple10 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - ( 6 ) -> G - ( 7 ) -> H - ( 8 ) -> I - ( 9 ) -> J - } - Tuple11 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - ( 6 ) -> G - ( 7 ) -> H - ( 8 ) -> I - ( 9 ) -> J - ( 10 ) -> K - } - Tuple12 { - ( 0 ) -> A - ( 1 ) -> B - ( 2 ) -> C - ( 3 ) -> D - ( 4 ) -> E - ( 5 ) -> F - ( 6 ) -> G - ( 7 ) -> H - ( 8 ) -> I - ( 9 ) -> J - ( 10 ) -> K - ( 11 ) -> L - } -}"#, - "fn foo () {}", - ); - } } diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 91b6db522..7411dd8b1 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -196,6 +196,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); + res.inner.insert( text.clone(), Binding::Simple(tt::Leaf::from(literal).into()), @@ -210,7 +211,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { - if input.eat_punct() != Some(punct) { + if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) { return Err(ExpandError::UnexpectedToken); } } @@ -246,8 +247,23 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result a.text == b.text, + (Literal(ref a), Literal(ref b)) => a.text == b.text, + (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { + let a_iter = a.iter().map(|a| a.char); + let b_iter = b.iter().map(|b| b.char); + a_iter.eq(b_iter) + } + _ => false, + }) + .unwrap_or(false) + { input.rollback(memento); break; } @@ -328,7 +344,7 @@ fn expand_tt( // Dirty hack to make macro-expansion terminate. // This should be replaced by a propper macro-by-example implementation let mut limit = 128; - let mut has_sep = false; + let mut has_seps = 0; while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) { limit -= 1; @@ -339,15 +355,28 @@ fn expand_tt( nesting.push(idx + 1); token_trees.push(reduce_single_token(t).into()); - if let Some(sep) = repeat.separator { - let punct = - tt::Leaf::from(tt::Punct { char: sep, spacing: tt::Spacing::Alone }); - token_trees.push(punct.into()); - has_sep = true; + if let Some(ref sep) = repeat.separator { + match sep { + crate::Separator::Ident(ident) => { + has_seps = 1; + token_trees.push(tt::Leaf::from(ident.clone()).into()); + } + crate::Separator::Literal(lit) => { + has_seps = 1; + token_trees.push(tt::Leaf::from(lit.clone()).into()); + } + + crate::Separator::Puncts(puncts) => { + has_seps = puncts.len(); + for punct in puncts { + token_trees.push(tt::Leaf::from(*punct).into()); + } + } + } } } nesting.pop().unwrap(); - if has_sep { + for _ in 0..has_seps { token_trees.pop(); } diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs index 0710062d9..c7ab463e2 100644 --- a/crates/ra_mbe/src/mbe_parser.rs +++ b/crates/ra_mbe/src/mbe_parser.rs @@ -74,18 +74,11 @@ fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result Result { - let subtree = p.eat_subtree().unwrap(); - let mut subtree = parse_subtree(subtree, transcriber)?; - subtree.delimiter = crate::Delimiter::None; - let sep = p.eat_punct().ok_or(ParseError::Expected(String::from("separator")))?; - let (separator, rep) = match sep.char { - '*' | '+' | '?' => (None, sep.char), - char => { - (Some(char), p.eat_punct().ok_or(ParseError::Expected(String::from("separator")))?.char) - } - }; - +fn mk_repeat( + rep: char, + subtree: crate::Subtree, + separator: Option, +) -> Result { let kind = match rep { '*' => crate::RepeatKind::ZeroOrMore, '+' => crate::RepeatKind::OneOrMore, @@ -95,6 +88,27 @@ fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result Result { + let subtree = p.eat_subtree().unwrap(); + let mut subtree = parse_subtree(subtree, transcriber)?; + subtree.delimiter = crate::Delimiter::None; + + if let Some(rep) = p.at_punct() { + match rep.char { + '*' | '+' | '?' => { + p.bump(); + return mk_repeat(rep.char, subtree, None); + } + _ => {} + } + } + + let sep = p.eat_seperator().ok_or(ParseError::Expected(String::from("separator")))?; + let rep = p.eat_punct().ok_or(ParseError::Expected(String::from("repeat")))?; + + mk_repeat(rep.char, subtree, Some(sep)) +} + #[cfg(test)] mod tests { use ra_syntax::{ast, AstNode}; @@ -109,7 +123,7 @@ mod tests { is_valid("($i:ident) => ()"); expect_err("$i:ident => ()", "subtree"); expect_err("($i:ident) ()", "`=`"); - expect_err("($($i:ident)_) => ()", "separator"); + expect_err("($($i:ident)_) => ()", "repeat"); } fn expect_err(macro_body: &str, expected: &str) { diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 3229cfa8f..e979777fe 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -342,7 +342,7 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> { } } -struct TokenPeek<'a, I> +pub(crate) struct TokenPeek<'a, I> where I: Iterator, { @@ -365,7 +365,7 @@ where TokenPeek { iter: itertools::multipeek(iter) } } - fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> { + pub fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> { if p.spacing != tt::Spacing::Joint { return None; } @@ -375,7 +375,7 @@ where Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint)) } - fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> { + pub fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> { self.current_punct2(p).and_then(|((p0, p1), last_joint)| { if !last_joint { None @@ -437,12 +437,16 @@ fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken { } fn convert_literal(l: &tt::Literal) -> TtToken { - TtToken { - kind: classify_literal(&l.text).unwrap().kind, - is_joint_to_next: false, - text: l.text.clone(), - n_tokens: 1, - } + let kind = classify_literal(&l.text) + .map(|tkn| tkn.kind) + .or_else(|| match l.text.as_ref() { + "true" => Some(SyntaxKind::TRUE_KW), + "false" => Some(SyntaxKind::FALSE_KW), + _ => None, + }) + .unwrap(); + + TtToken { kind, is_joint_to_next: false, text: l.text.clone(), n_tokens: 1 } } fn convert_ident(ident: &tt::Ident) -> TtToken { diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 38a481029..9cca19dbb 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -133,7 +133,9 @@ fn convert_tt( }; let mut token_trees = Vec::new(); - for child in tt.children_with_tokens().skip(skip_first as usize) { + let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); + + while let Some(child) = child_iter.next() { if (skip_first && (child == first_child || child == last_child)) || child.kind().is_trivia() { continue; @@ -152,12 +154,25 @@ fn convert_tt( prev = Some(char) } if let Some(char) = prev { - token_trees.push( - tt::Leaf::from(tt::Punct { char, spacing: tt::Spacing::Alone }).into(), - ); + let spacing = match child_iter.peek() { + Some(SyntaxElement::Token(token)) => { + if token.kind().is_punct() { + tt::Spacing::Joint + } else { + tt::Spacing::Alone + } + } + _ => tt::Spacing::Alone, + }; + + token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); } } else { - let child: tt::TokenTree = if token.kind().is_keyword() + let child: tt::TokenTree = if token.kind() == SyntaxKind::TRUE_KW + || token.kind() == SyntaxKind::FALSE_KW + { + tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() + } else if token.kind().is_keyword() || token.kind() == IDENT || token.kind() == LIFETIME { diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index 87bcf8b0d..343416421 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs @@ -1,5 +1,7 @@ use crate::ParseError; use crate::subtree_parser::Parser; +use crate::subtree_source::TokenPeek; +use smallvec::{SmallVec, smallvec}; #[derive(Clone)] pub(crate) struct TtCursor<'a> { @@ -162,6 +164,95 @@ impl<'a> TtCursor<'a> { } } + fn eat_punct3(&mut self, p: &tt::Punct) -> Option> { + let sec = self.eat_punct()?.clone(); + let third = self.eat_punct()?.clone(); + Some(smallvec![p.clone(), sec, third]) + } + + fn eat_punct2(&mut self, p: &tt::Punct) -> Option> { + let sec = self.eat_punct()?.clone(); + Some(smallvec![p.clone(), sec]) + } + + fn eat_multi_char_punct<'b, I>( + &mut self, + p: &tt::Punct, + iter: &mut TokenPeek<'b, I>, + ) -> Option> + where + I: Iterator, + { + if let Some((m, _)) = iter.current_punct3(p) { + if let r @ Some(_) = match m { + ('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => { + self.eat_punct3(p) + } + _ => None, + } { + return r; + } + } + + if let Some((m, _)) = iter.current_punct2(p) { + if let r @ Some(_) = match m { + ('<', '=') + | ('>', '=') + | ('+', '=') + | ('-', '=') + | ('|', '=') + | ('&', '=') + | ('^', '=') + | ('/', '=') + | ('*', '=') + | ('%', '=') + | ('&', '&') + | ('|', '|') + | ('<', '<') + | ('>', '>') + | ('-', '>') + | ('!', '=') + | ('=', '>') + | ('=', '=') + | ('.', '.') + | (':', ':') => self.eat_punct2(p), + + _ => None, + } { + return r; + } + } + + None + } + + pub(crate) fn eat_seperator(&mut self) -> Option { + match self.eat()? { + tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { + Some(crate::Separator::Literal(lit.clone())) + } + tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { + Some(crate::Separator::Ident(ident.clone())) + } + tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { + match punct.char { + '*' | '+' | '?' => return None, + _ => {} + }; + + // FIXME: The parser is only handle some compositeable punct, + // But at this phase, some punct still is jointed. + // So we by pass that check here. + let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter()); + let puncts = self.eat_multi_char_punct(punct, &mut peekable); + let puncts = puncts.unwrap_or_else(|| smallvec![punct.clone()]); + + Some(crate::Separator::Puncts(puncts)) + } + _ => None, + } + } + #[must_use] pub(crate) fn save(&self) -> TtCursorMemento { TtCursorMemento { pos: self.pos } -- cgit v1.2.3 From c55a2dbc1de8ba42df57b70f652eb6a0c0bbc9f6 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Thu, 25 Apr 2019 23:12:57 +0800 Subject: Fix more bugs --- crates/ra_mbe/Cargo.toml | 1 + crates/ra_mbe/src/lib.rs | 112 +++++++++++++++++++++++++++++++++++- crates/ra_mbe/src/mbe_expander.rs | 69 +++++++++++++++++----- crates/ra_mbe/src/subtree_source.rs | 9 +-- crates/ra_mbe/src/syntax_bridge.rs | 16 +++++- crates/ra_mbe/src/tt_cursor.rs | 2 +- 6 files changed, 186 insertions(+), 23 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml index 1e5ed6907..68f559295 100644 --- a/crates/ra_mbe/Cargo.toml +++ b/crates/ra_mbe/Cargo.toml @@ -11,3 +11,4 @@ tt = { path = "../ra_tt", package = "ra_tt" } itertools = "0.8.0" rustc-hash = "1.0.0" smallvec = "0.6.9" +log = "0.4.5" diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 7ebba807c..7817232d6 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -436,6 +436,30 @@ impl_froms!(TokenTree: Leaf, Subtree); assert_expansion(&rules, "foo! (fn baz {true true} )", "fn baz () -> bool {true &&true}"); } + #[test] + fn test_match_group_zero_match() { + let rules = create_rules( + r#" + macro_rules! foo { + ( $($i:ident)* ) => (); + }"#, + ); + + assert_expansion(&rules, "foo! ()", ""); + } + + #[test] + fn test_match_group_in_group() { + let rules = create_rules( + r#" + macro_rules! foo { + { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* ); + }"#, + ); + + assert_expansion(&rules, "foo! ( (a b) )", "(a b)"); + } + #[test] fn test_expand_to_item_list() { let rules = create_rules( @@ -1118,7 +1142,37 @@ macro_rules! impl_fn_for_zst { |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty $body: block; )+ } => { - fn foo(){} + $( + $( #[$attr] )* + struct $Name; + + impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name { + #[inline] + extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { + $body + } + } + + impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name { + #[inline] + extern "rust-call" fn call_mut( + &mut self, + ($( $arg, )*): ($( $ArgTy, )*) + ) -> $ReturnTy { + Fn::call(&*self, ($( $arg, )*)) + } + } + + impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name { + type Output = $ReturnTy; + + #[inline] + extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy { + Fn::call(&self, ($( $arg, )*)) + } + } + )+ +} } } "#, @@ -1141,7 +1195,7 @@ impl_fn_for_zst ! { } ; } "#, - "fn foo () {}"); + "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}"); } #[test] @@ -1160,4 +1214,58 @@ impl_fn_for_zst ! { assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#, "fn foo () {}"); } + + #[test] + fn test_cfg_if_items() { + // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986 + let rules = create_rules( + r#" + macro_rules! __cfg_if_items { + (($($not:meta,)*) ; ) => {}; + (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => { + __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* } + } + } +"#, + ); + + assert_expansion(&rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#, + "__cfg_if_items ! {(rustdoc , ) ; }"); + } + + #[test] + fn test_cfg_if_main() { + // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9 + let rules = create_rules( + r#" + macro_rules! cfg_if { + ($( + if #[cfg($($meta:meta),*)] { $($it:item)* } + ) else * else { + $($it2:item)* + }) => { + __cfg_if_items! { + () ; + $( ( ($($meta),*) ($($it)*) ), )* + ( () ($($it2)*) ), + } + } + } +"#, + ); + + assert_expansion(&rules, r#" +cfg_if ! { + if # [ cfg ( target_env = "msvc" ) ] { + // no extra unwinder support needed + } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] { + // no unwinder on the system! + } else { + mod libunwind ; + pub use libunwind :: * ; + } + } +"#, + "__cfg_if_items ! {() ; (() (mod libunwind ; pub use libunwind :: * ;)) ,}"); + } } diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 7411dd8b1..d5189b537 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs @@ -21,7 +21,10 @@ fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result Result { // Dirty hack to make macro-expansion terminate. // This should be replaced by a propper macro-by-example implementation - let mut limit = 128; + let mut limit = 65536; let mut counter = 0; let mut memento = input.save(); @@ -236,6 +239,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); break; } @@ -303,15 +307,21 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result { + bindings: &'a Bindings, + nesting: Vec, + var_expanded: bool, +} + fn expand_subtree( template: &crate::Subtree, - bindings: &Bindings, - nesting: &mut Vec, + ctx: &mut ExpandCtx, ) -> Result { let token_trees = template .token_trees .iter() - .map(|it| expand_tt(it, bindings, nesting)) + .map(|it| expand_tt(it, ctx)) .collect::, ExpandError>>()?; Ok(tt::Subtree { token_trees, delimiter: template.delimiter }) @@ -333,26 +343,43 @@ fn reduce_single_token(mut subtree: tt::Subtree) -> tt::TokenTree { fn expand_tt( template: &crate::TokenTree, - bindings: &Bindings, - nesting: &mut Vec, + ctx: &mut ExpandCtx, ) -> Result { let res: tt::TokenTree = match template { - crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(), + crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), crate::TokenTree::Repeat(repeat) => { let mut token_trees: Vec = Vec::new(); - nesting.push(0); + ctx.nesting.push(0); // Dirty hack to make macro-expansion terminate. // This should be replaced by a propper macro-by-example implementation - let mut limit = 128; + let mut limit = 65536; let mut has_seps = 0; + let mut counter = 0; + + let mut some_var_expanded = false; + ctx.var_expanded = false; + + while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { + // if no var expaned in the child, we count it as a fail + if !ctx.var_expanded { + break; + } + some_var_expanded = true; + ctx.var_expanded = false; - while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) { + counter += 1; limit -= 1; if limit == 0 { + log::warn!( + "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", + template, + ctx + ); break; } - let idx = nesting.pop().unwrap(); - nesting.push(idx + 1); + + let idx = ctx.nesting.pop().unwrap(); + ctx.nesting.push(idx + 1); token_trees.push(reduce_single_token(t).into()); if let Some(ref sep) = repeat.separator { @@ -374,12 +401,23 @@ fn expand_tt( } } } + + if let crate::RepeatKind::ZeroOrOne = repeat.kind { + break; + } } - nesting.pop().unwrap(); + + ctx.var_expanded = some_var_expanded; + + ctx.nesting.pop().unwrap(); for _ in 0..has_seps { token_trees.pop(); } + if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { + return Err(ExpandError::UnexpectedToken); + } + // Check if it is a singel token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None }) @@ -396,7 +434,8 @@ fn expand_tt( tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() }) .into() } else { - let tkn = bindings.get(&v.text, nesting)?.clone(); + let tkn = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); + ctx.var_expanded = true; if let tt::TokenTree::Subtree(subtree) = tkn { reduce_single_token(subtree) diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index e979777fe..845020f4e 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -212,7 +212,7 @@ impl<'a> SubTreeWalker<'a> { } pub(crate) trait Querier { - fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr); + fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool); } // A wrapper class for ref cell @@ -292,9 +292,10 @@ impl<'a> WalkerOwner<'a> { } impl<'a> Querier for WalkerOwner<'a> { - fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr) { - let tkn = self.get(uidx).unwrap(); - (tkn.kind, tkn.text) + fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) { + self.get(uidx) + .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next)) + .unwrap_or_else(|| (SyntaxKind::EOF, "".into(), false)) } } diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 9cca19dbb..e0f228ce9 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -123,6 +123,11 @@ fn convert_tt( global_offset: TextUnit, tt: &SyntaxNode, ) -> Option { + // This tree is empty + if tt.first_child_or_token().is_none() { + return Some(tt::Subtree { token_trees: vec![], delimiter: tt::Delimiter::None }); + } + let first_child = tt.first_child_or_token()?; let last_child = tt.last_child_or_token()?; let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { @@ -233,7 +238,16 @@ impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { self.text_pos += TextUnit::of_str(&self.buf); let text = SmolStr::new(self.buf.as_str()); self.buf.clear(); - self.inner.token(kind, text) + self.inner.token(kind, text); + + // // Add a white space to token + // let (last_kind, _, last_joint_to_next ) = self.src_querier.token(self.token_pos-n_tokens as usize); + // if !last_joint_to_next && last_kind.is_punct() { + // let (cur_kind, _, _ ) = self.src_querier.token(self.token_pos); + // if cur_kind.is_punct() { + // self.inner.token(WHITESPACE, " ".into()); + // } + // } } fn start_node(&mut self, kind: SyntaxKind) { diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs index 343416421..eef642a9c 100644 --- a/crates/ra_mbe/src/tt_cursor.rs +++ b/crates/ra_mbe/src/tt_cursor.rs @@ -3,7 +3,7 @@ use crate::subtree_parser::Parser; use crate::subtree_source::TokenPeek; use smallvec::{SmallVec, smallvec}; -#[derive(Clone)] +#[derive(Debug, Clone)] pub(crate) struct TtCursor<'a> { subtree: &'a tt::Subtree, pos: usize, -- cgit v1.2.3 From 1908819bf6432016527f4bd3b0f22500b85cab5f Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 26 Apr 2019 02:56:44 +0800 Subject: Use panic instead of unwrap --- crates/ra_mbe/src/subtree_source.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'crates/ra_mbe') diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 845020f4e..6255ea304 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs @@ -438,14 +438,12 @@ fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken { } fn convert_literal(l: &tt::Literal) -> TtToken { - let kind = classify_literal(&l.text) - .map(|tkn| tkn.kind) - .or_else(|| match l.text.as_ref() { - "true" => Some(SyntaxKind::TRUE_KW), - "false" => Some(SyntaxKind::FALSE_KW), - _ => None, - }) - .unwrap(); + let kind = + classify_literal(&l.text).map(|tkn| tkn.kind).unwrap_or_else(|| match l.text.as_ref() { + "true" => SyntaxKind::TRUE_KW, + "false" => SyntaxKind::FALSE_KW, + _ => panic!("Fail to convert given literal {:#?}", &l), + }); TtToken { kind, is_joint_to_next: false, text: l.text.clone(), n_tokens: 1 } } -- cgit v1.2.3