aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe
diff options
context:
space:
mode:
authorbors[bot] <bors[bot]@users.noreply.github.com>2019-04-25 19:59:34 +0100
committerbors[bot] <bors[bot]@users.noreply.github.com>2019-04-25 19:59:34 +0100
commit42c4e0f378faeabd425392d4a7a7839bd7e8ac2f (patch)
treef48081403383b7a00a3c297b2ff24a7c11ddd07e /crates/ra_mbe
parent5bbd9f43cc217a44445fde91b4b53ca85d78cd92 (diff)
parent1908819bf6432016527f4bd3b0f22500b85cab5f (diff)
Merge #1209
1209: Bugs fixes And Improvements of MBE r=matklad a=edwin0cheng This PR fixed / improve followings things: * Add `token` `$repeat` separator support: Previously $repeat only support single punct separator. * Fixed a bug which expand infinite pattern, see `test_match_group_in_group` * Correctly handle +,*,? case of $repeat patterns * Increase the limit of $repeat patterns (128 => 65536), personally i think we could remove this limit as we seem to fix all major loop bugs * **Re-enable tt matcher** * Better meta item parsing. * Add related tests and add some real world test cases. * Add more debug information. Co-authored-by: Edwin Cheng <[email protected]>
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r--crates/ra_mbe/Cargo.toml2
-rw-r--r--crates/ra_mbe/src/lib.rs216
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs167
-rw-r--r--crates/ra_mbe/src/mbe_parser.rs40
-rw-r--r--crates/ra_mbe/src/subtree_source.rs29
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs41
-rw-r--r--crates/ra_mbe/src/tt_cursor.rs106
7 files changed, 517 insertions, 84 deletions
diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml
index 1d0c2a340..68f559295 100644
--- a/crates/ra_mbe/Cargo.toml
+++ b/crates/ra_mbe/Cargo.toml
@@ -10,3 +10,5 @@ ra_parser = { path = "../ra_parser" }
10tt = { path = "../ra_tt", package = "ra_tt" } 10tt = { path = "../ra_tt", package = "ra_tt" }
11itertools = "0.8.0" 11itertools = "0.8.0"
12rustc-hash = "1.0.0" 12rustc-hash = "1.0.0"
13smallvec = "0.6.9"
14log = "0.4.5"
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index eedc0c5dd..7817232d6 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -24,6 +24,7 @@ mod subtree_source;
24mod subtree_parser; 24mod subtree_parser;
25 25
26use ra_syntax::SmolStr; 26use ra_syntax::SmolStr;
27use smallvec::SmallVec;
27 28
28pub use tt::{Delimiter, Punct}; 29pub use tt::{Delimiter, Punct};
29 30
@@ -99,10 +100,17 @@ pub(crate) struct Subtree {
99} 100}
100 101
101#[derive(Clone, Debug, PartialEq, Eq)] 102#[derive(Clone, Debug, PartialEq, Eq)]
103pub(crate) enum Separator {
104 Literal(tt::Literal),
105 Ident(tt::Ident),
106 Puncts(SmallVec<[tt::Punct; 3]>),
107}
108
109#[derive(Clone, Debug, PartialEq, Eq)]
102pub(crate) struct Repeat { 110pub(crate) struct Repeat {
103 pub(crate) subtree: Subtree, 111 pub(crate) subtree: Subtree,
104 pub(crate) kind: RepeatKind, 112 pub(crate) kind: RepeatKind,
105 pub(crate) separator: Option<char>, 113 pub(crate) separator: Option<Separator>,
106} 114}
107 115
108#[derive(Clone, Debug, PartialEq, Eq)] 116#[derive(Clone, Debug, PartialEq, Eq)]
@@ -175,8 +183,8 @@ impl_froms!(TokenTree: Leaf, Subtree);
175 let expansion = rules.expand(&invocation_tt).unwrap(); 183 let expansion = rules.expand(&invocation_tt).unwrap();
176 assert_eq!( 184 assert_eq!(
177 expansion.to_string(), 185 expansion.to_string(),
178 "impl From < Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree :: Leaf (it)}} \ 186 "impl From <Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \
179 impl From < Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree :: Subtree (it)}}" 187 impl From <Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}"
180 ) 188 )
181 } 189 }
182 190
@@ -384,7 +392,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
384"#, 392"#,
385 ); 393 );
386 394
387 assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}"); 395 assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ;bar ()}");
388 } 396 }
389 397
390 #[test] 398 #[test]
@@ -417,6 +425,42 @@ impl_froms!(TokenTree: Leaf, Subtree);
417 } 425 }
418 426
419 #[test] 427 #[test]
428 fn test_match_group_with_multichar_sep() {
429 let rules = create_rules(
430 r#"
431 macro_rules! foo {
432 (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} );
433 }"#,
434 );
435
436 assert_expansion(&rules, "foo! (fn baz {true true} )", "fn baz () -> bool {true &&true}");
437 }
438
439 #[test]
440 fn test_match_group_zero_match() {
441 let rules = create_rules(
442 r#"
443 macro_rules! foo {
444 ( $($i:ident)* ) => ();
445 }"#,
446 );
447
448 assert_expansion(&rules, "foo! ()", "");
449 }
450
451 #[test]
452 fn test_match_group_in_group() {
453 let rules = create_rules(
454 r#"
455 macro_rules! foo {
456 { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* );
457 }"#,
458 );
459
460 assert_expansion(&rules, "foo! ( (a b) )", "(a b)");
461 }
462
463 #[test]
420 fn test_expand_to_item_list() { 464 fn test_expand_to_item_list() {
421 let rules = create_rules( 465 let rules = create_rules(
422 " 466 "
@@ -597,7 +641,7 @@ MACRO_ITEMS@[0; 40)
597 assert_expansion( 641 assert_expansion(
598 &rules, 642 &rules,
599 "foo! { bar::<u8>::baz::<u8> }", 643 "foo! { bar::<u8>::baz::<u8> }",
600 "fn foo () {let a = bar :: < u8 > :: baz :: < u8 > ;}", 644 "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}",
601 ); 645 );
602 } 646 }
603 647
@@ -891,7 +935,7 @@ MACRO_ITEMS@[0; 40)
891 } 935 }
892"#, 936"#,
893 ); 937 );
894 assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref < 'a > {s : & 'a str}"#); 938 assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#);
895 } 939 }
896 940
897 #[test] 941 #[test]
@@ -1063,7 +1107,165 @@ macro_rules! int_base {
1063 ); 1107 );
1064 1108
1065 assert_expansion(&rules, r#" int_base!{Binary for isize as usize -> Binary}"#, 1109 assert_expansion(&rules, r#" int_base!{Binary for isize as usize -> Binary}"#,
1066 "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt :: Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}" 1110 "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
1111 );
1112 }
1113
1114 #[test]
1115 fn test_generate_pattern_iterators() {
1116 // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs
1117 let rules = create_rules(
1118 r#"
1119macro_rules! generate_pattern_iterators {
1120 { double ended; with $(#[$common_stability_attribute:meta])*,
1121 $forward_iterator:ident,
1122 $reverse_iterator:ident, $iterty:ty
1123 } => {
1124 fn foo(){}
1125 }
1126}
1127"#,
1128 );
1129
1130 assert_expansion(&rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str )"#,
1131 "fn foo () {}");
1132 }
1133
1134 #[test]
1135 fn test_impl_fn_for_zst() {
1136 // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs
1137 let rules = create_rules(
1138 r#"
1139macro_rules! impl_fn_for_zst {
1140 { $( $( #[$attr: meta] )*
1141 struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
1142 |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty
1143$body: block; )+
1144 } => {
1145 $(
1146 $( #[$attr] )*
1147 struct $Name;
1148
1149 impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name {
1150 #[inline]
1151 extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
1152 $body
1153 }
1154 }
1155
1156 impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name {
1157 #[inline]
1158 extern "rust-call" fn call_mut(
1159 &mut self,
1160 ($( $arg, )*): ($( $ArgTy, )*)
1161 ) -> $ReturnTy {
1162 Fn::call(&*self, ($( $arg, )*))
1163 }
1164 }
1165
1166 impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name {
1167 type Output = $ReturnTy;
1168
1169 #[inline]
1170 extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
1171 Fn::call(&self, ($( $arg, )*))
1172 }
1173 }
1174 )+
1175}
1176 }
1177}
1178"#,
1179 );
1180
1181 assert_expansion(&rules, r#"
1182impl_fn_for_zst ! {
1183 # [ derive ( Clone ) ]
1184 struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug {
1185 c . escape_debug_ext ( false )
1186 } ;
1187
1188 # [ derive ( Clone ) ]
1189 struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode {
1190 c . escape_unicode ( )
1191 } ;
1192 # [ derive ( Clone ) ]
1193 struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault {
1194 c . escape_default ( )
1195 } ;
1196 }
1197"#,
1198 "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}");
1199 }
1200
1201 #[test]
1202 fn test_impl_nonzero_fmt() {
1203 // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
1204 let rules = create_rules(
1205 r#"
1206 macro_rules! impl_nonzero_fmt {
1207 ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
1208 fn foo() {}
1209 }
1210 }
1211"#,
1067 ); 1212 );
1213
1214 assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#,
1215 "fn foo () {}");
1216 }
1217
1218 #[test]
1219 fn test_cfg_if_items() {
1220 // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986
1221 let rules = create_rules(
1222 r#"
1223 macro_rules! __cfg_if_items {
1224 (($($not:meta,)*) ; ) => {};
1225 (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => {
1226 __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* }
1227 }
1228 }
1229"#,
1230 );
1231
1232 assert_expansion(&rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#,
1233 "__cfg_if_items ! {(rustdoc , ) ; }");
1234 }
1235
1236 #[test]
1237 fn test_cfg_if_main() {
1238 // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9
1239 let rules = create_rules(
1240 r#"
1241 macro_rules! cfg_if {
1242 ($(
1243 if #[cfg($($meta:meta),*)] { $($it:item)* }
1244 ) else * else {
1245 $($it2:item)*
1246 }) => {
1247 __cfg_if_items! {
1248 () ;
1249 $( ( ($($meta),*) ($($it)*) ), )*
1250 ( () ($($it2)*) ),
1251 }
1252 }
1253 }
1254"#,
1255 );
1256
1257 assert_expansion(&rules, r#"
1258cfg_if ! {
1259 if # [ cfg ( target_env = "msvc" ) ] {
1260 // no extra unwinder support needed
1261 } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] {
1262 // no unwinder on the system!
1263 } else {
1264 mod libunwind ;
1265 pub use libunwind :: * ;
1266 }
1267 }
1268"#,
1269 "__cfg_if_items ! {() ; (() (mod libunwind ; pub use libunwind :: * ;)) ,}");
1068 } 1270 }
1069} 1271}
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 00fb09a3b..d5189b537 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -21,7 +21,10 @@ fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, E
21 if !input.is_eof() { 21 if !input.is_eof() {
22 return Err(ExpandError::UnexpectedToken); 22 return Err(ExpandError::UnexpectedToken);
23 } 23 }
24 expand_subtree(&rule.rhs, &bindings, &mut Vec::new()) 24
25 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
26
27 expand_subtree(&rule.rhs, &mut ctx)
25} 28}
26 29
27/// The actual algorithm for expansion is not too hard, but is pretty tricky. 30/// The actual algorithm for expansion is not too hard, but is pretty tricky.
@@ -179,10 +182,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
179 // Enable followiing code when everything is fixed 182 // Enable followiing code when everything is fixed
180 // At least we can dogfood itself to not stackoverflow 183 // At least we can dogfood itself to not stackoverflow
181 // 184 //
182 // "tt" => { 185 "tt" => {
183 // let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(); 186 let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
184 // res.inner.insert(text.clone(), Binding::Simple(token.into())); 187 res.inner.insert(text.clone(), Binding::Simple(token.into()));
185 // } 188 }
186 "item" => { 189 "item" => {
187 let item = 190 let item =
188 input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone(); 191 input.eat_item().ok_or(ExpandError::UnexpectedToken)?.clone();
@@ -196,6 +199,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
196 "literal" => { 199 "literal" => {
197 let literal = 200 let literal =
198 input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); 201 input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
202
199 res.inner.insert( 203 res.inner.insert(
200 text.clone(), 204 text.clone(),
201 Binding::Simple(tt::Leaf::from(literal).into()), 205 Binding::Simple(tt::Leaf::from(literal).into()),
@@ -210,7 +214,7 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
210 } 214 }
211 } 215 }
212 crate::Leaf::Punct(punct) => { 216 crate::Leaf::Punct(punct) => {
213 if input.eat_punct() != Some(punct) { 217 if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
214 return Err(ExpandError::UnexpectedToken); 218 return Err(ExpandError::UnexpectedToken);
215 } 219 }
216 } 220 }
@@ -224,20 +228,54 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
224 crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => { 228 crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
225 // Dirty hack to make macro-expansion terminate. 229 // Dirty hack to make macro-expansion terminate.
226 // This should be replaced by a propper macro-by-example implementation 230 // This should be replaced by a propper macro-by-example implementation
227 let mut limit = 128; 231 let mut limit = 65536;
228 let mut counter = 0; 232 let mut counter = 0;
229 while let Ok(nested) = match_lhs(subtree, input) { 233
230 counter += 1; 234 let mut memento = input.save();
231 limit -= 1; 235
232 if limit == 0 { 236 loop {
233 break; 237 match match_lhs(subtree, input) {
234 } 238 Ok(nested) => {
235 res.push_nested(nested)?; 239 counter += 1;
236 if let Some(separator) = *separator { 240 limit -= 1;
237 if !input.is_eof() { 241 if limit == 0 {
238 if input.eat_punct().map(|p| p.char) != Some(separator) { 242 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
239 return Err(ExpandError::UnexpectedToken); 243 break;
244 }
245
246 memento = input.save();
247 res.push_nested(nested)?;
248 if counter == 1 {
249 if let crate::RepeatKind::ZeroOrOne = kind {
250 break;
251 }
240 } 252 }
253
254 if let Some(separator) = separator {
255 use crate::Separator::*;
256
257 if !input
258 .eat_seperator()
259 .map(|sep| match (sep, separator) {
260 (Ident(ref a), Ident(ref b)) => a.text == b.text,
261 (Literal(ref a), Literal(ref b)) => a.text == b.text,
262 (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
263 let a_iter = a.iter().map(|a| a.char);
264 let b_iter = b.iter().map(|b| b.char);
265 a_iter.eq(b_iter)
266 }
267 _ => false,
268 })
269 .unwrap_or(false)
270 {
271 input.rollback(memento);
272 break;
273 }
274 }
275 }
276 Err(_) => {
277 input.rollback(memento);
278 break;
241 } 279 }
242 } 280 }
243 } 281 }
@@ -246,10 +284,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
246 crate::RepeatKind::OneOrMore if counter == 0 => { 284 crate::RepeatKind::OneOrMore if counter == 0 => {
247 return Err(ExpandError::UnexpectedToken); 285 return Err(ExpandError::UnexpectedToken);
248 } 286 }
249 crate::RepeatKind::ZeroOrOne if counter > 1 => {
250 return Err(ExpandError::UnexpectedToken);
251 }
252
253 _ => {} 287 _ => {}
254 } 288 }
255 } 289 }
@@ -273,15 +307,21 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
273 Ok(res) 307 Ok(res)
274} 308}
275 309
310#[derive(Debug)]
311struct ExpandCtx<'a> {
312 bindings: &'a Bindings,
313 nesting: Vec<usize>,
314 var_expanded: bool,
315}
316
276fn expand_subtree( 317fn expand_subtree(
277 template: &crate::Subtree, 318 template: &crate::Subtree,
278 bindings: &Bindings, 319 ctx: &mut ExpandCtx,
279 nesting: &mut Vec<usize>,
280) -> Result<tt::Subtree, ExpandError> { 320) -> Result<tt::Subtree, ExpandError> {
281 let token_trees = template 321 let token_trees = template
282 .token_trees 322 .token_trees
283 .iter() 323 .iter()
284 .map(|it| expand_tt(it, bindings, nesting)) 324 .map(|it| expand_tt(it, ctx))
285 .collect::<Result<Vec<_>, ExpandError>>()?; 325 .collect::<Result<Vec<_>, ExpandError>>()?;
286 326
287 Ok(tt::Subtree { token_trees, delimiter: template.delimiter }) 327 Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
@@ -303,43 +343,81 @@ fn reduce_single_token(mut subtree: tt::Subtree) -> tt::TokenTree {
303 343
304fn expand_tt( 344fn expand_tt(
305 template: &crate::TokenTree, 345 template: &crate::TokenTree,
306 bindings: &Bindings, 346 ctx: &mut ExpandCtx,
307 nesting: &mut Vec<usize>,
308) -> Result<tt::TokenTree, ExpandError> { 347) -> Result<tt::TokenTree, ExpandError> {
309 let res: tt::TokenTree = match template { 348 let res: tt::TokenTree = match template {
310 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, bindings, nesting)?.into(), 349 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
311 crate::TokenTree::Repeat(repeat) => { 350 crate::TokenTree::Repeat(repeat) => {
312 let mut token_trees: Vec<tt::TokenTree> = Vec::new(); 351 let mut token_trees: Vec<tt::TokenTree> = Vec::new();
313 nesting.push(0); 352 ctx.nesting.push(0);
314 // Dirty hack to make macro-expansion terminate. 353 // Dirty hack to make macro-expansion terminate.
315 // This should be replaced by a propper macro-by-example implementation 354 // This should be replaced by a propper macro-by-example implementation
316 let mut limit = 128; 355 let mut limit = 65536;
317 let mut has_sep = false; 356 let mut has_seps = 0;
357 let mut counter = 0;
318 358
319 while let Ok(t) = expand_subtree(&repeat.subtree, bindings, nesting) { 359 let mut some_var_expanded = false;
360 ctx.var_expanded = false;
361
362 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
363 // if no var expaned in the child, we count it as a fail
364 if !ctx.var_expanded {
365 break;
366 }
367 some_var_expanded = true;
368 ctx.var_expanded = false;
369
370 counter += 1;
320 limit -= 1; 371 limit -= 1;
321 if limit == 0 { 372 if limit == 0 {
373 log::warn!(
374 "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
375 template,
376 ctx
377 );
322 break; 378 break;
323 } 379 }
324 let idx = nesting.pop().unwrap(); 380
325 nesting.push(idx + 1); 381 let idx = ctx.nesting.pop().unwrap();
382 ctx.nesting.push(idx + 1);
326 token_trees.push(reduce_single_token(t).into()); 383 token_trees.push(reduce_single_token(t).into());
327 384
328 if let Some(sep) = repeat.separator { 385 if let Some(ref sep) = repeat.separator {
329 let punct = 386 match sep {
330 tt::Leaf::from(tt::Punct { char: sep, spacing: tt::Spacing::Alone }); 387 crate::Separator::Ident(ident) => {
331 token_trees.push(punct.into()); 388 has_seps = 1;
332 has_sep = true; 389 token_trees.push(tt::Leaf::from(ident.clone()).into());
390 }
391 crate::Separator::Literal(lit) => {
392 has_seps = 1;
393 token_trees.push(tt::Leaf::from(lit.clone()).into());
394 }
395
396 crate::Separator::Puncts(puncts) => {
397 has_seps = puncts.len();
398 for punct in puncts {
399 token_trees.push(tt::Leaf::from(*punct).into());
400 }
401 }
402 }
403 }
404
405 if let crate::RepeatKind::ZeroOrOne = repeat.kind {
406 break;
333 } 407 }
334 } 408 }
335 nesting.pop().unwrap();
336 409
337 // Dirty hack for remove the last sep 410 ctx.var_expanded = some_var_expanded;
338 // if it is a "," undo the push 411
339 if has_sep && repeat.separator.unwrap() == ',' { 412 ctx.nesting.pop().unwrap();
413 for _ in 0..has_seps {
340 token_trees.pop(); 414 token_trees.pop();
341 } 415 }
342 416
417 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
418 return Err(ExpandError::UnexpectedToken);
419 }
420
343 // Check if it is a singel token subtree without any delimiter 421 // Check if it is a singel token subtree without any delimiter
344 // e.g {Delimiter:None> ['>'] /Delimiter:None>} 422 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
345 reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None }) 423 reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None })
@@ -356,7 +434,8 @@ fn expand_tt(
356 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() }) 434 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
357 .into() 435 .into()
358 } else { 436 } else {
359 let tkn = bindings.get(&v.text, nesting)?.clone(); 437 let tkn = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
438 ctx.var_expanded = true;
360 439
361 if let tt::TokenTree::Subtree(subtree) = tkn { 440 if let tt::TokenTree::Subtree(subtree) = tkn {
362 reduce_single_token(subtree) 441 reduce_single_token(subtree)
diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs
index 0710062d9..c7ab463e2 100644
--- a/crates/ra_mbe/src/mbe_parser.rs
+++ b/crates/ra_mbe/src/mbe_parser.rs
@@ -74,18 +74,11 @@ fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseErr
74 Ok(crate::Var { text, kind }) 74 Ok(crate::Var { text, kind })
75} 75}
76 76
77fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> { 77fn mk_repeat(
78 let subtree = p.eat_subtree().unwrap(); 78 rep: char,
79 let mut subtree = parse_subtree(subtree, transcriber)?; 79 subtree: crate::Subtree,
80 subtree.delimiter = crate::Delimiter::None; 80 separator: Option<crate::Separator>,
81 let sep = p.eat_punct().ok_or(ParseError::Expected(String::from("separator")))?; 81) -> Result<crate::Repeat, ParseError> {
82 let (separator, rep) = match sep.char {
83 '*' | '+' | '?' => (None, sep.char),
84 char => {
85 (Some(char), p.eat_punct().ok_or(ParseError::Expected(String::from("separator")))?.char)
86 }
87 };
88
89 let kind = match rep { 82 let kind = match rep {
90 '*' => crate::RepeatKind::ZeroOrMore, 83 '*' => crate::RepeatKind::ZeroOrMore,
91 '+' => crate::RepeatKind::OneOrMore, 84 '+' => crate::RepeatKind::OneOrMore,
@@ -95,6 +88,27 @@ fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, Pa
95 Ok(crate::Repeat { subtree, kind, separator }) 88 Ok(crate::Repeat { subtree, kind, separator })
96} 89}
97 90
91fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> {
92 let subtree = p.eat_subtree().unwrap();
93 let mut subtree = parse_subtree(subtree, transcriber)?;
94 subtree.delimiter = crate::Delimiter::None;
95
96 if let Some(rep) = p.at_punct() {
97 match rep.char {
98 '*' | '+' | '?' => {
99 p.bump();
100 return mk_repeat(rep.char, subtree, None);
101 }
102 _ => {}
103 }
104 }
105
106 let sep = p.eat_seperator().ok_or(ParseError::Expected(String::from("separator")))?;
107 let rep = p.eat_punct().ok_or(ParseError::Expected(String::from("repeat")))?;
108
109 mk_repeat(rep.char, subtree, Some(sep))
110}
111
98#[cfg(test)] 112#[cfg(test)]
99mod tests { 113mod tests {
100 use ra_syntax::{ast, AstNode}; 114 use ra_syntax::{ast, AstNode};
@@ -109,7 +123,7 @@ mod tests {
109 is_valid("($i:ident) => ()"); 123 is_valid("($i:ident) => ()");
110 expect_err("$i:ident => ()", "subtree"); 124 expect_err("$i:ident => ()", "subtree");
111 expect_err("($i:ident) ()", "`=`"); 125 expect_err("($i:ident) ()", "`=`");
112 expect_err("($($i:ident)_) => ()", "separator"); 126 expect_err("($($i:ident)_) => ()", "repeat");
113 } 127 }
114 128
115 fn expect_err(macro_body: &str, expected: &str) { 129 fn expect_err(macro_body: &str, expected: &str) {
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 3229cfa8f..6255ea304 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -212,7 +212,7 @@ impl<'a> SubTreeWalker<'a> {
212} 212}
213 213
214pub(crate) trait Querier { 214pub(crate) trait Querier {
215 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr); 215 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool);
216} 216}
217 217
218// A wrapper class for ref cell 218// A wrapper class for ref cell
@@ -292,9 +292,10 @@ impl<'a> WalkerOwner<'a> {
292} 292}
293 293
294impl<'a> Querier for WalkerOwner<'a> { 294impl<'a> Querier for WalkerOwner<'a> {
295 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr) { 295 fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr, bool) {
296 let tkn = self.get(uidx).unwrap(); 296 self.get(uidx)
297 (tkn.kind, tkn.text) 297 .map(|tkn| (tkn.kind, tkn.text, tkn.is_joint_to_next))
298 .unwrap_or_else(|| (SyntaxKind::EOF, "".into(), false))
298 } 299 }
299} 300}
300 301
@@ -342,7 +343,7 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
342 } 343 }
343} 344}
344 345
345struct TokenPeek<'a, I> 346pub(crate) struct TokenPeek<'a, I>
346where 347where
347 I: Iterator<Item = &'a tt::TokenTree>, 348 I: Iterator<Item = &'a tt::TokenTree>,
348{ 349{
@@ -365,7 +366,7 @@ where
365 TokenPeek { iter: itertools::multipeek(iter) } 366 TokenPeek { iter: itertools::multipeek(iter) }
366 } 367 }
367 368
368 fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> { 369 pub fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> {
369 if p.spacing != tt::Spacing::Joint { 370 if p.spacing != tt::Spacing::Joint {
370 return None; 371 return None;
371 } 372 }
@@ -375,7 +376,7 @@ where
375 Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint)) 376 Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
376 } 377 }
377 378
378 fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> { 379 pub fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> {
379 self.current_punct2(p).and_then(|((p0, p1), last_joint)| { 380 self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
380 if !last_joint { 381 if !last_joint {
381 None 382 None
@@ -437,12 +438,14 @@ fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken {
437} 438}
438 439
439fn convert_literal(l: &tt::Literal) -> TtToken { 440fn convert_literal(l: &tt::Literal) -> TtToken {
440 TtToken { 441 let kind =
441 kind: classify_literal(&l.text).unwrap().kind, 442 classify_literal(&l.text).map(|tkn| tkn.kind).unwrap_or_else(|| match l.text.as_ref() {
442 is_joint_to_next: false, 443 "true" => SyntaxKind::TRUE_KW,
443 text: l.text.clone(), 444 "false" => SyntaxKind::FALSE_KW,
444 n_tokens: 1, 445 _ => panic!("Fail to convert given literal {:#?}", &l),
445 } 446 });
447
448 TtToken { kind, is_joint_to_next: false, text: l.text.clone(), n_tokens: 1 }
446} 449}
447 450
448fn convert_ident(ident: &tt::Ident) -> TtToken { 451fn convert_ident(ident: &tt::Ident) -> TtToken {
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 38a481029..e0f228ce9 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -123,6 +123,11 @@ fn convert_tt(
123 global_offset: TextUnit, 123 global_offset: TextUnit,
124 tt: &SyntaxNode, 124 tt: &SyntaxNode,
125) -> Option<tt::Subtree> { 125) -> Option<tt::Subtree> {
126 // This tree is empty
127 if tt.first_child_or_token().is_none() {
128 return Some(tt::Subtree { token_trees: vec![], delimiter: tt::Delimiter::None });
129 }
130
126 let first_child = tt.first_child_or_token()?; 131 let first_child = tt.first_child_or_token()?;
127 let last_child = tt.last_child_or_token()?; 132 let last_child = tt.last_child_or_token()?;
128 let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { 133 let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) {
@@ -133,7 +138,9 @@ fn convert_tt(
133 }; 138 };
134 139
135 let mut token_trees = Vec::new(); 140 let mut token_trees = Vec::new();
136 for child in tt.children_with_tokens().skip(skip_first as usize) { 141 let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
142
143 while let Some(child) = child_iter.next() {
137 if (skip_first && (child == first_child || child == last_child)) || child.kind().is_trivia() 144 if (skip_first && (child == first_child || child == last_child)) || child.kind().is_trivia()
138 { 145 {
139 continue; 146 continue;
@@ -152,12 +159,25 @@ fn convert_tt(
152 prev = Some(char) 159 prev = Some(char)
153 } 160 }
154 if let Some(char) = prev { 161 if let Some(char) = prev {
155 token_trees.push( 162 let spacing = match child_iter.peek() {
156 tt::Leaf::from(tt::Punct { char, spacing: tt::Spacing::Alone }).into(), 163 Some(SyntaxElement::Token(token)) => {
157 ); 164 if token.kind().is_punct() {
165 tt::Spacing::Joint
166 } else {
167 tt::Spacing::Alone
168 }
169 }
170 _ => tt::Spacing::Alone,
171 };
172
173 token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into());
158 } 174 }
159 } else { 175 } else {
160 let child: tt::TokenTree = if token.kind().is_keyword() 176 let child: tt::TokenTree = if token.kind() == SyntaxKind::TRUE_KW
177 || token.kind() == SyntaxKind::FALSE_KW
178 {
179 tt::Leaf::from(tt::Literal { text: token.text().clone() }).into()
180 } else if token.kind().is_keyword()
161 || token.kind() == IDENT 181 || token.kind() == IDENT
162 || token.kind() == LIFETIME 182 || token.kind() == LIFETIME
163 { 183 {
@@ -218,7 +238,16 @@ impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> {
218 self.text_pos += TextUnit::of_str(&self.buf); 238 self.text_pos += TextUnit::of_str(&self.buf);
219 let text = SmolStr::new(self.buf.as_str()); 239 let text = SmolStr::new(self.buf.as_str());
220 self.buf.clear(); 240 self.buf.clear();
221 self.inner.token(kind, text) 241 self.inner.token(kind, text);
242
243 // // Add a white space to token
244 // let (last_kind, _, last_joint_to_next ) = self.src_querier.token(self.token_pos-n_tokens as usize);
245 // if !last_joint_to_next && last_kind.is_punct() {
246 // let (cur_kind, _, _ ) = self.src_querier.token(self.token_pos);
247 // if cur_kind.is_punct() {
248 // self.inner.token(WHITESPACE, " ".into());
249 // }
250 // }
222 } 251 }
223 252
224 fn start_node(&mut self, kind: SyntaxKind) { 253 fn start_node(&mut self, kind: SyntaxKind) {
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs
index 741b5ea1c..eef642a9c 100644
--- a/crates/ra_mbe/src/tt_cursor.rs
+++ b/crates/ra_mbe/src/tt_cursor.rs
@@ -1,12 +1,18 @@
1use crate::ParseError; 1use crate::ParseError;
2use crate::subtree_parser::Parser; 2use crate::subtree_parser::Parser;
3use crate::subtree_source::TokenPeek;
4use smallvec::{SmallVec, smallvec};
3 5
4#[derive(Clone)] 6#[derive(Debug, Clone)]
5pub(crate) struct TtCursor<'a> { 7pub(crate) struct TtCursor<'a> {
6 subtree: &'a tt::Subtree, 8 subtree: &'a tt::Subtree,
7 pos: usize, 9 pos: usize,
8} 10}
9 11
12pub(crate) struct TtCursorMemento {
13 pos: usize,
14}
15
10impl<'a> TtCursor<'a> { 16impl<'a> TtCursor<'a> {
11 pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> { 17 pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> {
12 TtCursor { subtree, pos: 0 } 18 TtCursor { subtree, pos: 0 }
@@ -157,4 +163,102 @@ impl<'a> TtCursor<'a> {
157 Err(ParseError::Expected(format!("`{}`", char))) 163 Err(ParseError::Expected(format!("`{}`", char)))
158 } 164 }
159 } 165 }
166
167 fn eat_punct3(&mut self, p: &tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
168 let sec = self.eat_punct()?.clone();
169 let third = self.eat_punct()?.clone();
170 Some(smallvec![p.clone(), sec, third])
171 }
172
173 fn eat_punct2(&mut self, p: &tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
174 let sec = self.eat_punct()?.clone();
175 Some(smallvec![p.clone(), sec])
176 }
177
178 fn eat_multi_char_punct<'b, I>(
179 &mut self,
180 p: &tt::Punct,
181 iter: &mut TokenPeek<'b, I>,
182 ) -> Option<SmallVec<[tt::Punct; 3]>>
183 where
184 I: Iterator<Item = &'b tt::TokenTree>,
185 {
186 if let Some((m, _)) = iter.current_punct3(p) {
187 if let r @ Some(_) = match m {
188 ('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => {
189 self.eat_punct3(p)
190 }
191 _ => None,
192 } {
193 return r;
194 }
195 }
196
197 if let Some((m, _)) = iter.current_punct2(p) {
198 if let r @ Some(_) = match m {
199 ('<', '=')
200 | ('>', '=')
201 | ('+', '=')
202 | ('-', '=')
203 | ('|', '=')
204 | ('&', '=')
205 | ('^', '=')
206 | ('/', '=')
207 | ('*', '=')
208 | ('%', '=')
209 | ('&', '&')
210 | ('|', '|')
211 | ('<', '<')
212 | ('>', '>')
213 | ('-', '>')
214 | ('!', '=')
215 | ('=', '>')
216 | ('=', '=')
217 | ('.', '.')
218 | (':', ':') => self.eat_punct2(p),
219
220 _ => None,
221 } {
222 return r;
223 }
224 }
225
226 None
227 }
228
229 pub(crate) fn eat_seperator(&mut self) -> Option<crate::Separator> {
230 match self.eat()? {
231 tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
232 Some(crate::Separator::Literal(lit.clone()))
233 }
234 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
235 Some(crate::Separator::Ident(ident.clone()))
236 }
237 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
238 match punct.char {
239 '*' | '+' | '?' => return None,
240 _ => {}
241 };
242
243 // FIXME: The parser is only handle some compositeable punct,
244 // But at this phase, some punct still is jointed.
245 // So we by pass that check here.
246 let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter());
247 let puncts = self.eat_multi_char_punct(punct, &mut peekable);
248 let puncts = puncts.unwrap_or_else(|| smallvec![punct.clone()]);
249
250 Some(crate::Separator::Puncts(puncts))
251 }
252 _ => None,
253 }
254 }
255
256 #[must_use]
257 pub(crate) fn save(&self) -> TtCursorMemento {
258 TtCursorMemento { pos: self.pos }
259 }
260
261 pub(crate) fn rollback(&mut self, memento: TtCursorMemento) {
262 self.pos = memento.pos;
263 }
160} 264}