aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe/src
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_mbe/src')
-rw-r--r--crates/ra_mbe/src/lib.rs1158
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs152
-rw-r--r--crates/ra_mbe/src/mbe_parser.rs38
-rw-r--r--crates/ra_mbe/src/subtree_source.rs223
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs137
-rw-r--r--crates/ra_mbe/src/tests.rs1399
-rw-r--r--crates/ra_mbe/src/tt_cursor.rs57
7 files changed, 1773 insertions, 1391 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index be9ea3ebb..c146252a4 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -15,7 +15,6 @@ macro_rules! impl_froms {
15 } 15 }
16} 16}
17 17
18// mod tt_cursor;
19mod mbe_parser; 18mod mbe_parser;
20mod mbe_expander; 19mod mbe_expander;
21mod syntax_bridge; 20mod syntax_bridge;
@@ -99,13 +98,31 @@ pub(crate) struct Subtree {
99 pub(crate) token_trees: Vec<TokenTree>, 98 pub(crate) token_trees: Vec<TokenTree>,
100} 99}
101 100
102#[derive(Clone, Debug, PartialEq, Eq)] 101#[derive(Clone, Debug, Eq)]
103pub(crate) enum Separator { 102pub(crate) enum Separator {
104 Literal(tt::Literal), 103 Literal(tt::Literal),
105 Ident(tt::Ident), 104 Ident(tt::Ident),
106 Puncts(SmallVec<[tt::Punct; 3]>), 105 Puncts(SmallVec<[tt::Punct; 3]>),
107} 106}
108 107
108// Note that when we compare a Separator, we just care about its textual value.
109impl PartialEq for crate::Separator {
110 fn eq(&self, other: &crate::Separator) -> bool {
111 use crate::Separator::*;
112
113 match (self, other) {
114 (Ident(ref a), Ident(ref b)) => a.text == b.text,
115 (Literal(ref a), Literal(ref b)) => a.text == b.text,
116 (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
117 let a_iter = a.iter().map(|a| a.char);
118 let b_iter = b.iter().map(|b| b.char);
119 a_iter.eq(b_iter)
120 }
121 _ => false,
122 }
123 }
124}
125
109#[derive(Clone, Debug, PartialEq, Eq)] 126#[derive(Clone, Debug, PartialEq, Eq)]
110pub(crate) struct Repeat { 127pub(crate) struct Repeat {
111 pub(crate) subtree: Subtree, 128 pub(crate) subtree: Subtree,
@@ -137,1139 +154,4 @@ pub(crate) struct Var {
137} 154}
138 155
139#[cfg(test)] 156#[cfg(test)]
140mod tests { 157mod tests;
141 use ra_syntax::{ast, AstNode};
142
143 use super::*;
144
145 // Good first issue (although a slightly challenging one):
146 //
147 // * Pick a random test from here
148 // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
149 // * Port the test to rust and add it to this module
150 // * Make it pass :-)
151
152 #[test]
153 fn test_convert_tt() {
154 let macro_definition = r#"
155macro_rules! impl_froms {
156 ($e:ident: $($v:ident),*) => {
157 $(
158 impl From<$v> for $e {
159 fn from(it: $v) -> $e {
160 $e::$v(it)
161 }
162 }
163 )*
164 }
165}
166"#;
167
168 let macro_invocation = r#"
169impl_froms!(TokenTree: Leaf, Subtree);
170"#;
171
172 let source_file = ast::SourceFile::parse(macro_definition);
173 let macro_definition =
174 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
175
176 let source_file = ast::SourceFile::parse(macro_invocation);
177 let macro_invocation =
178 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
179
180 let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
181 let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
182 let rules = crate::MacroRules::parse(&definition_tt).unwrap();
183 let expansion = rules.expand(&invocation_tt).unwrap();
184 assert_eq!(
185 expansion.to_string(),
186 "impl From <Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \
187 impl From <Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}"
188 )
189 }
190
191 pub(crate) fn create_rules(macro_definition: &str) -> MacroRules {
192 let source_file = ast::SourceFile::parse(macro_definition);
193 let macro_definition =
194 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
195
196 let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
197 crate::MacroRules::parse(&definition_tt).unwrap()
198 }
199
200 pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
201 let source_file = ast::SourceFile::parse(invocation);
202 let macro_invocation =
203 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
204
205 let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
206
207 rules.expand(&invocation_tt).unwrap()
208 }
209
210 pub(crate) fn expand_to_items(
211 rules: &MacroRules,
212 invocation: &str,
213 ) -> ra_syntax::TreeArc<ast::MacroItems> {
214 let expanded = expand(rules, invocation);
215 token_tree_to_macro_items(&expanded).unwrap()
216 }
217
218 #[allow(unused)]
219 pub(crate) fn expand_to_stmts(
220 rules: &MacroRules,
221 invocation: &str,
222 ) -> ra_syntax::TreeArc<ast::MacroStmts> {
223 let expanded = expand(rules, invocation);
224 token_tree_to_macro_stmts(&expanded).unwrap()
225 }
226
227 pub(crate) fn expand_to_expr(
228 rules: &MacroRules,
229 invocation: &str,
230 ) -> ra_syntax::TreeArc<ast::Expr> {
231 let expanded = expand(rules, invocation);
232 token_tree_to_expr(&expanded).unwrap()
233 }
234
235 pub(crate) fn assert_expansion(
236 rules: &MacroRules,
237 invocation: &str,
238 expansion: &str,
239 ) -> tt::Subtree {
240 let expanded = expand(rules, invocation);
241 assert_eq!(expanded.to_string(), expansion);
242
243 // FIXME: Temp comment below code
244 // It is because after the lexer change,
245 // The SyntaxNode structure cannot be matched easily
246
247 // let tree = token_tree_to_macro_items(&expanded);
248
249 // // Eat all white space by parse it back and forth
250 // // Because $crate will seperate in two token , will do some special treatment here
251 // let expansion = expansion.replace("$crate", "C_C__C");
252 // let expansion = ast::SourceFile::parse(&expansion);
253 // let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0;
254 // let file = token_tree_to_macro_items(&expansion);
255 // let file = file.unwrap().syntax().debug_dump().trim().to_string();
256 // let tree = tree.unwrap().syntax().debug_dump().trim().to_string();
257
258 // let file = file.replace("C_C__C", "$crate");
259 // assert_eq!(tree, file,);
260
261 expanded
262 }
263
264 #[test]
265 fn test_fail_match_pattern_by_first_token() {
266 let rules = create_rules(
267 r#"
268 macro_rules! foo {
269 ($ i:ident) => (
270 mod $ i {}
271 );
272 (= $ i:ident) => (
273 fn $ i() {}
274 );
275 (+ $ i:ident) => (
276 struct $ i;
277 )
278 }
279"#,
280 );
281
282 assert_expansion(&rules, "foo! { foo }", "mod foo {}");
283 assert_expansion(&rules, "foo! { = bar }", "fn bar () {}");
284 assert_expansion(&rules, "foo! { + Baz }", "struct Baz ;");
285 }
286
287 #[test]
288 fn test_fail_match_pattern_by_last_token() {
289 let rules = create_rules(
290 r#"
291 macro_rules! foo {
292 ($ i:ident) => (
293 mod $ i {}
294 );
295 ($ i:ident =) => (
296 fn $ i() {}
297 );
298 ($ i:ident +) => (
299 struct $ i;
300 )
301 }
302"#,
303 );
304
305 assert_expansion(&rules, "foo! { foo }", "mod foo {}");
306 assert_expansion(&rules, "foo! { bar = }", "fn bar () {}");
307 assert_expansion(&rules, "foo! { Baz + }", "struct Baz ;");
308 }
309
310 #[test]
311 fn test_fail_match_pattern_by_word_token() {
312 let rules = create_rules(
313 r#"
314 macro_rules! foo {
315 ($ i:ident) => (
316 mod $ i {}
317 );
318 (spam $ i:ident) => (
319 fn $ i() {}
320 );
321 (eggs $ i:ident) => (
322 struct $ i;
323 )
324 }
325"#,
326 );
327
328 assert_expansion(&rules, "foo! { foo }", "mod foo {}");
329 assert_expansion(&rules, "foo! { spam bar }", "fn bar () {}");
330 assert_expansion(&rules, "foo! { eggs Baz }", "struct Baz ;");
331 }
332
333 #[test]
334 fn test_match_group_pattern_by_separator_token() {
335 let rules = create_rules(
336 r#"
337 macro_rules! foo {
338 ($ ($ i:ident),*) => ($ (
339 mod $ i {}
340 )*);
341 ($ ($ i:ident)#*) => ($ (
342 fn $ i() {}
343 )*);
344 ($ i:ident ,# $ j:ident) => (
345 struct $ i;
346 struct $ j;
347 )
348 }
349"#,
350 );
351
352 assert_expansion(&rules, "foo! { foo, bar }", "mod foo {} mod bar {}");
353 assert_expansion(&rules, "foo! { foo# bar }", "fn foo () {} fn bar () {}");
354 assert_expansion(&rules, "foo! { Foo,# Bar }", "struct Foo ; struct Bar ;");
355 }
356
357 #[test]
358 fn test_match_group_pattern_with_multiple_defs() {
359 let rules = create_rules(
360 r#"
361 macro_rules! foo {
362 ($ ($ i:ident),*) => ( struct Bar { $ (
363 fn $ i {}
364 )*} );
365 }
366"#,
367 );
368
369 assert_expansion(&rules, "foo! { foo, bar }", "struct Bar {fn foo {} fn bar {}}");
370 }
371
372 #[test]
373 fn test_match_group_pattern_with_multiple_statement() {
374 let rules = create_rules(
375 r#"
376 macro_rules! foo {
377 ($ ($ i:ident),*) => ( fn baz { $ (
378 $ i ();
379 )*} );
380 }
381"#,
382 );
383
384 assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}");
385 }
386
387 #[test]
388 fn test_match_group_pattern_with_multiple_statement_without_semi() {
389 let rules = create_rules(
390 r#"
391 macro_rules! foo {
392 ($ ($ i:ident),*) => ( fn baz { $ (
393 $i()
394 );*} );
395 }
396"#,
397 );
398
399 assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ;bar ()}");
400 }
401
402 #[test]
403 fn test_match_group_empty_fixed_token() {
404 let rules = create_rules(
405 r#"
406 macro_rules! foo {
407 ($ ($ i:ident)* #abc) => ( fn baz { $ (
408 $ i ();
409 )*} );
410 }
411"#,
412 );
413
414 assert_expansion(&rules, "foo! {#abc}", "fn baz {}");
415 }
416
417 #[test]
418 fn test_match_group_in_subtree() {
419 let rules = create_rules(
420 r#"
421 macro_rules! foo {
422 (fn $name:ident {$($i:ident)*} ) => ( fn $name() { $ (
423 $ i ();
424 )*} );
425 }"#,
426 );
427
428 assert_expansion(&rules, "foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}");
429 }
430
431 #[test]
432 fn test_match_group_with_multichar_sep() {
433 let rules = create_rules(
434 r#"
435 macro_rules! foo {
436 (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} );
437 }"#,
438 );
439
440 assert_expansion(&rules, "foo! (fn baz {true true} )", "fn baz () -> bool {true &&true}");
441 }
442
443 #[test]
444 fn test_match_group_zero_match() {
445 let rules = create_rules(
446 r#"
447 macro_rules! foo {
448 ( $($i:ident)* ) => ();
449 }"#,
450 );
451
452 assert_expansion(&rules, "foo! ()", "");
453 }
454
455 #[test]
456 fn test_match_group_in_group() {
457 let rules = create_rules(
458 r#"
459 macro_rules! foo {
460 { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* );
461 }"#,
462 );
463
464 assert_expansion(&rules, "foo! ( (a b) )", "(a b)");
465 }
466
467 #[test]
468 fn test_expand_to_item_list() {
469 let rules = create_rules(
470 "
471 macro_rules! structs {
472 ($($i:ident),*) => {
473 $(struct $i { field: u32 } )*
474 }
475 }
476 ",
477 );
478 let expansion = expand(&rules, "structs!(Foo, Bar)");
479 let tree = token_tree_to_macro_items(&expansion);
480 assert_eq!(
481 tree.unwrap().syntax().debug_dump().trim(),
482 r#"
483MACRO_ITEMS@[0; 40)
484 STRUCT_DEF@[0; 20)
485 STRUCT_KW@[0; 6) "struct"
486 NAME@[6; 9)
487 IDENT@[6; 9) "Foo"
488 NAMED_FIELD_DEF_LIST@[9; 20)
489 L_CURLY@[9; 10) "{"
490 NAMED_FIELD_DEF@[10; 19)
491 NAME@[10; 15)
492 IDENT@[10; 15) "field"
493 COLON@[15; 16) ":"
494 PATH_TYPE@[16; 19)
495 PATH@[16; 19)
496 PATH_SEGMENT@[16; 19)
497 NAME_REF@[16; 19)
498 IDENT@[16; 19) "u32"
499 R_CURLY@[19; 20) "}"
500 STRUCT_DEF@[20; 40)
501 STRUCT_KW@[20; 26) "struct"
502 NAME@[26; 29)
503 IDENT@[26; 29) "Bar"
504 NAMED_FIELD_DEF_LIST@[29; 40)
505 L_CURLY@[29; 30) "{"
506 NAMED_FIELD_DEF@[30; 39)
507 NAME@[30; 35)
508 IDENT@[30; 35) "field"
509 COLON@[35; 36) ":"
510 PATH_TYPE@[36; 39)
511 PATH@[36; 39)
512 PATH_SEGMENT@[36; 39)
513 NAME_REF@[36; 39)
514 IDENT@[36; 39) "u32"
515 R_CURLY@[39; 40) "}""#
516 .trim()
517 );
518 }
519
520 #[test]
521 fn test_expand_literals_to_token_tree() {
522 fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
523 if let tt::TokenTree::Subtree(subtree) = tt {
524 return &subtree;
525 }
526 unreachable!("It is not a subtree");
527 }
528
529 fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
530 if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
531 return lit;
532 }
533 unreachable!("It is not a literal");
534 }
535
536 let rules = create_rules(
537 r#"
538 macro_rules! literals {
539 ($i:ident) => {
540 {
541 let a = 'c';
542 let c = 1000;
543 let f = 12E+99_f64;
544 let s = "rust1";
545 }
546 }
547 }
548 "#,
549 );
550 let expansion = expand(&rules, "literals!(foo)");
551 let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees;
552
553 // [let] [a] [=] ['c'] [;]
554 assert_eq!(to_literal(&stm_tokens[3]).text, "'c'");
555 // [let] [c] [=] [1000] [;]
556 assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000");
557 // [let] [f] [=] [12E+99_f64] [;]
558 assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64");
559 // [let] [s] [=] ["rust1"] [;]
560 assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
561 }
562
563 #[test]
564 fn test_two_idents() {
565 let rules = create_rules(
566 r#"
567 macro_rules! foo {
568 ($ i:ident, $ j:ident) => {
569 fn foo() { let a = $ i; let b = $j; }
570 }
571 }
572"#,
573 );
574 assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
575 }
576
577 #[test]
578 fn test_tt_to_stmts() {
579 let rules = create_rules(
580 r#"
581 macro_rules! foo {
582 () => {
583 let a = 0;
584 a = 10 + 1;
585 a
586 }
587 }
588"#,
589 );
590
591 let expanded = expand(&rules, "foo!{}");
592 let stmts = token_tree_to_macro_stmts(&expanded);
593
594 assert_eq!(
595 stmts.unwrap().syntax().debug_dump().trim(),
596 r#"MACRO_STMTS@[0; 15)
597 LET_STMT@[0; 7)
598 LET_KW@[0; 3) "let"
599 BIND_PAT@[3; 4)
600 NAME@[3; 4)
601 IDENT@[3; 4) "a"
602 EQ@[4; 5) "="
603 LITERAL@[5; 6)
604 INT_NUMBER@[5; 6) "0"
605 SEMI@[6; 7) ";"
606 EXPR_STMT@[7; 14)
607 BIN_EXPR@[7; 13)
608 PATH_EXPR@[7; 8)
609 PATH@[7; 8)
610 PATH_SEGMENT@[7; 8)
611 NAME_REF@[7; 8)
612 IDENT@[7; 8) "a"
613 EQ@[8; 9) "="
614 BIN_EXPR@[9; 13)
615 LITERAL@[9; 11)
616 INT_NUMBER@[9; 11) "10"
617 PLUS@[11; 12) "+"
618 LITERAL@[12; 13)
619 INT_NUMBER@[12; 13) "1"
620 SEMI@[13; 14) ";"
621 EXPR_STMT@[14; 15)
622 PATH_EXPR@[14; 15)
623 PATH@[14; 15)
624 PATH_SEGMENT@[14; 15)
625 NAME_REF@[14; 15)
626 IDENT@[14; 15) "a""#,
627 );
628 }
629
630 // The following tests are port from intellij-rust directly
631 // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
632
633 #[test]
634 fn test_path() {
635 let rules = create_rules(
636 r#"
637 macro_rules! foo {
638 ($ i:path) => {
639 fn foo() { let a = $ i; }
640 }
641 }
642"#,
643 );
644 assert_expansion(&rules, "foo! { foo }", "fn foo () {let a = foo ;}");
645 assert_expansion(
646 &rules,
647 "foo! { bar::<u8>::baz::<u8> }",
648 "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}",
649 );
650 }
651
652 #[test]
653 fn test_two_paths() {
654 let rules = create_rules(
655 r#"
656 macro_rules! foo {
657 ($ i:path, $ j:path) => {
658 fn foo() { let a = $ i; let b = $j; }
659 }
660 }
661"#,
662 );
663 assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}");
664 }
665
666 #[test]
667 fn test_path_with_path() {
668 let rules = create_rules(
669 r#"
670 macro_rules! foo {
671 ($ i:path) => {
672 fn foo() { let a = $ i :: bar; }
673 }
674 }
675"#,
676 );
677 assert_expansion(&rules, "foo! { foo }", "fn foo () {let a = foo :: bar ;}");
678 }
679
680 #[test]
681 fn test_expr() {
682 let rules = create_rules(
683 r#"
684 macro_rules! foo {
685 ($ i:expr) => {
686 fn bar() { $ i; }
687 }
688 }
689"#,
690 );
691
692 assert_expansion(
693 &rules,
694 "foo! { 2 + 2 * baz(3).quux() }",
695 "fn bar () {2 + 2 * baz (3) . quux () ;}",
696 );
697 }
698
699 #[test]
700 fn test_expr_order() {
701 let rules = create_rules(
702 r#"
703 macro_rules! foo {
704 ($ i:expr) => {
705 fn bar() { $ i * 2; }
706 }
707 }
708"#,
709 );
710
711 assert_eq!(
712 expand_to_items(&rules, "foo! { 1 + 1 }").syntax().debug_dump().trim(),
713 r#"MACRO_ITEMS@[0; 15)
714 FN_DEF@[0; 15)
715 FN_KW@[0; 2) "fn"
716 NAME@[2; 5)
717 IDENT@[2; 5) "bar"
718 PARAM_LIST@[5; 7)
719 L_PAREN@[5; 6) "("
720 R_PAREN@[6; 7) ")"
721 BLOCK@[7; 15)
722 L_CURLY@[7; 8) "{"
723 EXPR_STMT@[8; 14)
724 BIN_EXPR@[8; 13)
725 BIN_EXPR@[8; 11)
726 LITERAL@[8; 9)
727 INT_NUMBER@[8; 9) "1"
728 PLUS@[9; 10) "+"
729 LITERAL@[10; 11)
730 INT_NUMBER@[10; 11) "1"
731 STAR@[11; 12) "*"
732 LITERAL@[12; 13)
733 INT_NUMBER@[12; 13) "2"
734 SEMI@[13; 14) ";"
735 R_CURLY@[14; 15) "}""#,
736 );
737 }
738
739 #[test]
740 fn test_last_expr() {
741 let rules = create_rules(
742 r#"
743 macro_rules! vec {
744 ($($item:expr),*) => {
745 {
746 let mut v = Vec::new();
747 $(
748 v.push($item);
749 )*
750 v
751 }
752 };
753 }
754"#,
755 );
756 assert_expansion(
757 &rules,
758 "vec!(1,2,3)",
759 "{let mut v = Vec :: new () ; v . push (1) ; v . push (2) ; v . push (3) ; v}",
760 );
761 }
762
763 #[test]
764 fn test_ty() {
765 let rules = create_rules(
766 r#"
767 macro_rules! foo {
768 ($ i:ty) => (
769 fn bar() -> $ i { unimplemented!() }
770 )
771 }
772"#,
773 );
774 assert_expansion(
775 &rules,
776 "foo! { Baz<u8> }",
777 "fn bar () -> Baz < u8 > {unimplemented ! ()}",
778 );
779 }
780
781 #[test]
782 fn test_ty_with_complex_type() {
783 let rules = create_rules(
784 r#"
785 macro_rules! foo {
786 ($ i:ty) => (
787 fn bar() -> $ i { unimplemented!() }
788 )
789 }
790"#,
791 );
792
793 // Reference lifetime struct with generic type
794 assert_expansion(
795 &rules,
796 "foo! { &'a Baz<u8> }",
797 "fn bar () -> & 'a Baz < u8 > {unimplemented ! ()}",
798 );
799
800 // extern "Rust" func type
801 assert_expansion(
802 &rules,
803 r#"foo! { extern "Rust" fn() -> Ret }"#,
804 r#"fn bar () -> extern "Rust" fn () -> Ret {unimplemented ! ()}"#,
805 );
806 }
807
808 #[test]
809 fn test_pat_() {
810 let rules = create_rules(
811 r#"
812 macro_rules! foo {
813 ($ i:pat) => { fn foo() { let $ i; } }
814 }
815"#,
816 );
817 assert_expansion(&rules, "foo! { (a, b) }", "fn foo () {let (a , b) ;}");
818 }
819
820 #[test]
821 fn test_stmt() {
822 let rules = create_rules(
823 r#"
824 macro_rules! foo {
825 ($ i:stmt) => (
826 fn bar() { $ i; }
827 )
828 }
829"#,
830 );
831 assert_expansion(&rules, "foo! { 2 }", "fn bar () {2 ;}");
832 assert_expansion(&rules, "foo! { let a = 0 }", "fn bar () {let a = 0 ;}");
833 }
834
835 #[test]
836 fn test_single_item() {
837 let rules = create_rules(
838 r#"
839 macro_rules! foo {
840 ($ i:item) => (
841 $ i
842 )
843 }
844"#,
845 );
846 assert_expansion(&rules, "foo! {mod c {}}", "mod c {}");
847 }
848
849 #[test]
850 fn test_all_items() {
851 let rules = create_rules(
852 r#"
853 macro_rules! foo {
854 ($ ($ i:item)*) => ($ (
855 $ i
856 )*)
857 }
858"#,
859 );
860 assert_expansion(&rules, r#"
861 foo! {
862 extern crate a;
863 mod b;
864 mod c {}
865 use d;
866 const E: i32 = 0;
867 static F: i32 = 0;
868 impl G {}
869 struct H;
870 enum I { Foo }
871 trait J {}
872 fn h() {}
873 extern {}
874 type T = u8;
875 }
876"#, r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#);
877 }
878
879 #[test]
880 fn test_block() {
881 let rules = create_rules(
882 r#"
883 macro_rules! foo {
884 ($ i:block) => { fn foo() $ i }
885 }
886"#,
887 );
888 assert_expansion(&rules, "foo! { { 1; } }", "fn foo () {1 ;}");
889 }
890
891 #[test]
892 fn test_meta() {
893 let rules = create_rules(
894 r#"
895 macro_rules! foo {
896 ($ i:meta) => (
897 #[$ i]
898 fn bar() {}
899 )
900 }
901"#,
902 );
903 assert_expansion(
904 &rules,
905 r#"foo! { cfg(target_os = "windows") }"#,
906 r#"# [cfg (target_os = "windows")] fn bar () {}"#,
907 );
908 }
909
910 #[test]
911 // fn test_tt_block() {
912 // let rules = create_rules(
913 // r#"
914 // macro_rules! foo {
915 // ($ i:tt) => { fn foo() $ i }
916 // }
917 // "#,
918 // );
919 // assert_expansion(&rules, r#"foo! { { 1; } }"#, r#"fn foo () {1 ;}"#);
920 // }
921
922 // #[test]
923 // fn test_tt_group() {
924 // let rules = create_rules(
925 // r#"
926 // macro_rules! foo {
927 // ($($ i:tt)*) => { $($ i)* }
928 // }
929 // "#,
930 // );
931 // assert_expansion(&rules, r#"foo! { fn foo() {} }"#, r#"fn foo () {}"#);
932 // }
933 #[test]
934 fn test_lifetime() {
935 let rules = create_rules(
936 r#"
937 macro_rules! foo {
938 ($ lt:lifetime) => { struct Ref<$ lt>{ s: &$ lt str } }
939 }
940"#,
941 );
942 assert_expansion(&rules, r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#);
943 }
944
945 #[test]
946 fn test_literal() {
947 let rules = create_rules(
948 r#"
949 macro_rules! foo {
950 ($ type:ty $ lit:literal) => { const VALUE: $ type = $ lit;};
951 }
952"#,
953 );
954 assert_expansion(&rules, r#"foo!(u8 0)"#, r#"const VALUE : u8 = 0 ;"#);
955 }
956
957 #[test]
958 fn test_vis() {
959 let rules = create_rules(
960 r#"
961 macro_rules! foo {
962 ($ vis:vis $ name:ident) => { $ vis fn $ name() {}};
963 }
964"#,
965 );
966 assert_expansion(&rules, r#"foo!(pub foo);"#, r#"pub fn foo () {}"#);
967 }
968
969 // The following tests are based on real world situations
970 #[test]
971 fn test_vec() {
972 let rules = create_rules(
973 r#"
974 macro_rules! vec {
975 ($($item:expr),*) => {
976 {
977 let mut v = Vec::new();
978 $(
979 v.push($item);
980 )*
981 v
982 }
983 };
984}
985"#,
986 );
987 assert_expansion(&rules, r#"vec!();"#, r#"{let mut v = Vec :: new () ; v}"#);
988 assert_expansion(
989 &rules,
990 r#"vec![1u32,2]"#,
991 r#"{let mut v = Vec :: new () ; v . push (1u32) ; v . push (2) ; v}"#,
992 );
993
994 assert_eq!(
995 expand_to_expr(&rules, r#"vec![1u32,2]"#).syntax().debug_dump().trim(),
996 r#"BLOCK_EXPR@[0; 45)
997 BLOCK@[0; 45)
998 L_CURLY@[0; 1) "{"
999 LET_STMT@[1; 20)
1000 LET_KW@[1; 4) "let"
1001 BIND_PAT@[4; 8)
1002 MUT_KW@[4; 7) "mut"
1003 NAME@[7; 8)
1004 IDENT@[7; 8) "v"
1005 EQ@[8; 9) "="
1006 CALL_EXPR@[9; 19)
1007 PATH_EXPR@[9; 17)
1008 PATH@[9; 17)
1009 PATH@[9; 12)
1010 PATH_SEGMENT@[9; 12)
1011 NAME_REF@[9; 12)
1012 IDENT@[9; 12) "Vec"
1013 COLONCOLON@[12; 14) "::"
1014 PATH_SEGMENT@[14; 17)
1015 NAME_REF@[14; 17)
1016 IDENT@[14; 17) "new"
1017 ARG_LIST@[17; 19)
1018 L_PAREN@[17; 18) "("
1019 R_PAREN@[18; 19) ")"
1020 SEMI@[19; 20) ";"
1021 EXPR_STMT@[20; 33)
1022 METHOD_CALL_EXPR@[20; 32)
1023 PATH_EXPR@[20; 21)
1024 PATH@[20; 21)
1025 PATH_SEGMENT@[20; 21)
1026 NAME_REF@[20; 21)
1027 IDENT@[20; 21) "v"
1028 DOT@[21; 22) "."
1029 NAME_REF@[22; 26)
1030 IDENT@[22; 26) "push"
1031 ARG_LIST@[26; 32)
1032 L_PAREN@[26; 27) "("
1033 LITERAL@[27; 31)
1034 INT_NUMBER@[27; 31) "1u32"
1035 R_PAREN@[31; 32) ")"
1036 SEMI@[32; 33) ";"
1037 EXPR_STMT@[33; 43)
1038 METHOD_CALL_EXPR@[33; 42)
1039 PATH_EXPR@[33; 34)
1040 PATH@[33; 34)
1041 PATH_SEGMENT@[33; 34)
1042 NAME_REF@[33; 34)
1043 IDENT@[33; 34) "v"
1044 DOT@[34; 35) "."
1045 NAME_REF@[35; 39)
1046 IDENT@[35; 39) "push"
1047 ARG_LIST@[39; 42)
1048 L_PAREN@[39; 40) "("
1049 LITERAL@[40; 41)
1050 INT_NUMBER@[40; 41) "2"
1051 R_PAREN@[41; 42) ")"
1052 SEMI@[42; 43) ";"
1053 PATH_EXPR@[43; 44)
1054 PATH@[43; 44)
1055 PATH_SEGMENT@[43; 44)
1056 NAME_REF@[43; 44)
1057 IDENT@[43; 44) "v"
1058 R_CURLY@[44; 45) "}""#
1059 );
1060 }
1061
1062 #[test]
1063 fn test_winapi_struct() {
1064 // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/macros.rs#L366
1065
1066 let rules = create_rules(
1067 r#"
1068macro_rules! STRUCT {
1069 ($(#[$attrs:meta])* struct $name:ident {
1070 $($field:ident: $ftype:ty,)+
1071 }) => (
1072 #[repr(C)] #[derive(Copy)] $(#[$attrs])*
1073 pub struct $name {
1074 $(pub $field: $ftype,)+
1075 }
1076 impl Clone for $name {
1077 #[inline]
1078 fn clone(&self) -> $name { *self }
1079 }
1080 #[cfg(feature = "impl-default")]
1081 impl Default for $name {
1082 #[inline]
1083 fn default() -> $name { unsafe { $crate::_core::mem::zeroed() } }
1084 }
1085 );
1086}
1087"#,
1088 );
1089 // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/shared/d3d9caps.rs
1090 assert_expansion(&rules, r#"STRUCT!{struct D3DVSHADERCAPS2_0 {Caps: u8,}}"#,
1091 "# [repr (C)] # [derive (Copy)] pub struct D3DVSHADERCAPS2_0 {pub Caps : u8 ,} impl Clone for D3DVSHADERCAPS2_0 {# [inline] fn clone (& self) -> D3DVSHADERCAPS2_0 {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DVSHADERCAPS2_0 {# [inline] fn default () -> D3DVSHADERCAPS2_0 {unsafe {$crate :: _core :: mem :: zeroed ()}}}");
1092 assert_expansion(&rules, r#"STRUCT!{#[cfg_attr(target_arch = "x86", repr(packed))] struct D3DCONTENTPROTECTIONCAPS {Caps : u8 ,}}"#,
1093 "# [repr (C)] # [derive (Copy)] # [cfg_attr (target_arch = \"x86\" , repr (packed))] pub struct D3DCONTENTPROTECTIONCAPS {pub Caps : u8 ,} impl Clone for D3DCONTENTPROTECTIONCAPS {# [inline] fn clone (& self) -> D3DCONTENTPROTECTIONCAPS {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DCONTENTPROTECTIONCAPS {# [inline] fn default () -> D3DCONTENTPROTECTIONCAPS {unsafe {$crate :: _core :: mem :: zeroed ()}}}");
1094 }
1095
1096 #[test]
1097 fn test_int_base() {
1098 let rules = create_rules(
1099 r#"
1100macro_rules! int_base {
1101 ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => {
1102 #[stable(feature = "rust1", since = "1.0.0")]
1103 impl fmt::$Trait for $T {
1104 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1105 $Radix.fmt_int(*self as $U, f)
1106 }
1107 }
1108 }
1109}
1110"#,
1111 );
1112
1113 assert_expansion(&rules, r#" int_base!{Binary for isize as usize -> Binary}"#,
1114 "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
1115 );
1116 }
1117
1118 #[test]
1119 fn test_generate_pattern_iterators() {
1120 // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs
1121 let rules = create_rules(
1122 r#"
1123macro_rules! generate_pattern_iterators {
1124 { double ended; with $(#[$common_stability_attribute:meta])*,
1125 $forward_iterator:ident,
1126 $reverse_iterator:ident, $iterty:ty
1127 } => {
1128 fn foo(){}
1129 }
1130}
1131"#,
1132 );
1133
1134 assert_expansion(&rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str )"#,
1135 "fn foo () {}");
1136 }
1137
1138 #[test]
1139 fn test_impl_fn_for_zst() {
1140 // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs
1141 let rules = create_rules(
1142 r#"
1143macro_rules! impl_fn_for_zst {
1144 { $( $( #[$attr: meta] )*
1145 struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
1146 |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty
1147$body: block; )+
1148 } => {
1149 $(
1150 $( #[$attr] )*
1151 struct $Name;
1152
1153 impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name {
1154 #[inline]
1155 extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
1156 $body
1157 }
1158 }
1159
1160 impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name {
1161 #[inline]
1162 extern "rust-call" fn call_mut(
1163 &mut self,
1164 ($( $arg, )*): ($( $ArgTy, )*)
1165 ) -> $ReturnTy {
1166 Fn::call(&*self, ($( $arg, )*))
1167 }
1168 }
1169
1170 impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name {
1171 type Output = $ReturnTy;
1172
1173 #[inline]
1174 extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
1175 Fn::call(&self, ($( $arg, )*))
1176 }
1177 }
1178 )+
1179}
1180 }
1181}
1182"#,
1183 );
1184
1185 assert_expansion(&rules, r#"
1186impl_fn_for_zst ! {
1187 # [ derive ( Clone ) ]
1188 struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug {
1189 c . escape_debug_ext ( false )
1190 } ;
1191
1192 # [ derive ( Clone ) ]
1193 struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode {
1194 c . escape_unicode ( )
1195 } ;
1196 # [ derive ( Clone ) ]
1197 struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault {
1198 c . escape_default ( )
1199 } ;
1200 }
1201"#,
1202 "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}");
1203 }
1204
1205 #[test]
1206 fn test_impl_nonzero_fmt() {
1207 // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
1208 let rules = create_rules(
1209 r#"
1210 macro_rules! impl_nonzero_fmt {
1211 ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
1212 fn foo() {}
1213 }
1214 }
1215"#,
1216 );
1217
1218 assert_expansion(&rules, r#"impl_nonzero_fmt ! { # [ stable ( feature = "nonzero" , since = "1.28.0" ) ] ( Debug , Display , Binary , Octal , LowerHex , UpperHex ) for NonZeroU8 }"#,
1219 "fn foo () {}");
1220 }
1221
1222 #[test]
1223 fn test_cfg_if_items() {
1224 // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986
1225 let rules = create_rules(
1226 r#"
1227 macro_rules! __cfg_if_items {
1228 (($($not:meta,)*) ; ) => {};
1229 (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => {
1230 __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* }
1231 }
1232 }
1233"#,
1234 );
1235
1236 assert_expansion(&rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#,
1237 "__cfg_if_items ! {(rustdoc , ) ; }");
1238 }
1239
1240 #[test]
1241 fn test_cfg_if_main() {
1242 // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9
1243 let rules = create_rules(
1244 r#"
1245 macro_rules! cfg_if {
1246 ($(
1247 if #[cfg($($meta:meta),*)] { $($it:item)* }
1248 ) else * else {
1249 $($it2:item)*
1250 }) => {
1251 __cfg_if_items! {
1252 () ;
1253 $( ( ($($meta),*) ($($it)*) ), )*
1254 ( () ($($it2)*) ),
1255 }
1256 }
1257 }
1258"#,
1259 );
1260
1261 assert_expansion(&rules, r#"
1262cfg_if ! {
1263 if # [ cfg ( target_env = "msvc" ) ] {
1264 // no extra unwinder support needed
1265 } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] {
1266 // no unwinder on the system!
1267 } else {
1268 mod libunwind ;
1269 pub use libunwind :: * ;
1270 }
1271 }
1272"#,
1273 "__cfg_if_items ! {() ; (() (mod libunwind ; pub use libunwind :: * ;)) ,}");
1274 }
1275}
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index d5189b537..4b007647c 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -81,9 +81,29 @@ struct Bindings {
81enum Binding { 81enum Binding {
82 Simple(tt::TokenTree), 82 Simple(tt::TokenTree),
83 Nested(Vec<Binding>), 83 Nested(Vec<Binding>),
84 Empty,
84} 85}
85 86
86impl Bindings { 87impl Bindings {
88 fn push_optional(&mut self, name: &SmolStr) {
89 // FIXME: Do we have a better way to represent an empty token ?
90 // Insert an empty subtree for empty token
91 self.inner.insert(
92 name.clone(),
93 Binding::Simple(
94 tt::Subtree { delimiter: tt::Delimiter::None, token_trees: vec![] }.into(),
95 ),
96 );
97 }
98
99 fn push_empty(&mut self, name: &SmolStr) {
100 self.inner.insert(name.clone(), Binding::Empty);
101 }
102
103 fn contains(&self, name: &SmolStr) -> bool {
104 self.inner.contains_key(name)
105 }
106
87 fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree, ExpandError> { 107 fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&tt::TokenTree, ExpandError> {
88 let mut b = self 108 let mut b = self
89 .inner 109 .inner
@@ -96,6 +116,12 @@ impl Bindings {
96 "could not find nested binding `{}`", 116 "could not find nested binding `{}`",
97 name 117 name
98 )))?, 118 )))?,
119 Binding::Empty => {
120 return Err(ExpandError::BindingError(format!(
121 "could not find empty binding `{}`",
122 name
123 )))
124 }
99 }; 125 };
100 } 126 }
101 match b { 127 match b {
@@ -104,16 +130,26 @@ impl Bindings {
104 "expected simple binding, found nested binding `{}`", 130 "expected simple binding, found nested binding `{}`",
105 name 131 name
106 ))), 132 ))),
133 Binding::Empty => Err(ExpandError::BindingError(format!(
134 "expected simple binding, found empty binding `{}`",
135 name
136 ))),
107 } 137 }
108 } 138 }
109 139
110 fn push_nested(&mut self, nested: Bindings) -> Result<(), ExpandError> { 140 fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
111 for (key, value) in nested.inner { 141 for (key, value) in nested.inner {
112 if !self.inner.contains_key(&key) { 142 if !self.inner.contains_key(&key) {
113 self.inner.insert(key.clone(), Binding::Nested(Vec::new())); 143 self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
114 } 144 }
115 match self.inner.get_mut(&key) { 145 match self.inner.get_mut(&key) {
116 Some(Binding::Nested(it)) => it.push(value), 146 Some(Binding::Nested(it)) => {
147 // insert empty nested bindings before this one
148 while it.len() < idx {
149 it.push(Binding::Nested(vec![]));
150 }
151 it.push(value);
152 }
117 _ => { 153 _ => {
118 return Err(ExpandError::BindingError(format!( 154 return Err(ExpandError::BindingError(format!(
119 "could not find binding `{}`", 155 "could not find binding `{}`",
@@ -130,6 +166,24 @@ impl Bindings {
130 } 166 }
131} 167}
132 168
169fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
170 let mut res = vec![];
171
172 for tkn in subtree.token_trees.iter() {
173 match tkn {
174 crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
175 res.push(text.clone());
176 }
177 crate::TokenTree::Subtree(subtree) => {
178 res.extend(collect_vars(subtree));
179 }
180 _ => {}
181 }
182 }
183
184 res
185}
186
133fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> { 187fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
134 let mut res = Bindings::default(); 188 let mut res = Bindings::default();
135 for pat in pattern.token_trees.iter() { 189 for pat in pattern.token_trees.iter() {
@@ -178,10 +232,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
178 input.eat_meta().ok_or(ExpandError::UnexpectedToken)?.clone(); 232 input.eat_meta().ok_or(ExpandError::UnexpectedToken)?.clone();
179 res.inner.insert(text.clone(), Binding::Simple(meta.into())); 233 res.inner.insert(text.clone(), Binding::Simple(meta.into()));
180 } 234 }
181 // FIXME:
182 // Enable followiing code when everything is fixed
183 // At least we can dogfood itself to not stackoverflow
184 //
185 "tt" => { 235 "tt" => {
186 let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(); 236 let token = input.eat().ok_or(ExpandError::UnexpectedToken)?.clone();
187 res.inner.insert(text.clone(), Binding::Simple(token.into())); 237 res.inner.insert(text.clone(), Binding::Simple(token.into()));
@@ -206,8 +256,13 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
206 ); 256 );
207 } 257 }
208 "vis" => { 258 "vis" => {
209 let vis = input.eat_vis().ok_or(ExpandError::UnexpectedToken)?.clone(); 259 // `vis` is optional
210 res.inner.insert(text.clone(), Binding::Simple(vis.into())); 260 if let Some(vis) = input.try_eat_vis() {
261 let vis = vis.clone();
262 res.inner.insert(text.clone(), Binding::Simple(vis.into()));
263 } else {
264 res.push_optional(&text);
265 }
211 } 266 }
212 267
213 _ => return Err(ExpandError::UnexpectedToken), 268 _ => return Err(ExpandError::UnexpectedToken),
@@ -236,7 +291,6 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
236 loop { 291 loop {
237 match match_lhs(subtree, input) { 292 match match_lhs(subtree, input) {
238 Ok(nested) => { 293 Ok(nested) => {
239 counter += 1;
240 limit -= 1; 294 limit -= 1;
241 if limit == 0 { 295 if limit == 0 {
242 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); 296 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
@@ -244,7 +298,8 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
244 } 298 }
245 299
246 memento = input.save(); 300 memento = input.save();
247 res.push_nested(nested)?; 301 res.push_nested(counter, nested)?;
302 counter += 1;
248 if counter == 1 { 303 if counter == 1 {
249 if let crate::RepeatKind::ZeroOrOne = kind { 304 if let crate::RepeatKind::ZeroOrOne = kind {
250 break; 305 break;
@@ -252,20 +307,9 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
252 } 307 }
253 308
254 if let Some(separator) = separator { 309 if let Some(separator) = separator {
255 use crate::Separator::*;
256
257 if !input 310 if !input
258 .eat_seperator() 311 .eat_seperator()
259 .map(|sep| match (sep, separator) { 312 .map(|sep| sep == *separator)
260 (Ident(ref a), Ident(ref b)) => a.text == b.text,
261 (Literal(ref a), Literal(ref b)) => a.text == b.text,
262 (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
263 let a_iter = a.iter().map(|a| a.char);
264 let b_iter = b.iter().map(|b| b.char);
265 a_iter.eq(b_iter)
266 }
267 _ => false,
268 })
269 .unwrap_or(false) 313 .unwrap_or(false)
270 { 314 {
271 input.rollback(memento); 315 input.rollback(memento);
@@ -284,6 +328,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
284 crate::RepeatKind::OneOrMore if counter == 0 => { 328 crate::RepeatKind::OneOrMore if counter == 0 => {
285 return Err(ExpandError::UnexpectedToken); 329 return Err(ExpandError::UnexpectedToken);
286 } 330 }
331 _ if counter == 0 => {
332 // Collect all empty variables in subtrees
333 collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
334 }
287 _ => {} 335 _ => {}
288 } 336 }
289 } 337 }
@@ -322,6 +370,14 @@ fn expand_subtree(
322 .token_trees 370 .token_trees
323 .iter() 371 .iter()
324 .map(|it| expand_tt(it, ctx)) 372 .map(|it| expand_tt(it, ctx))
373 .filter(|it| {
374 // Filter empty subtree
375 if let Ok(tt::TokenTree::Subtree(subtree)) = it {
376 subtree.delimiter != tt::Delimiter::None || !subtree.token_trees.is_empty()
377 } else {
378 true
379 }
380 })
325 .collect::<Result<Vec<_>, ExpandError>>()?; 381 .collect::<Result<Vec<_>, ExpandError>>()?;
326 382
327 Ok(tt::Subtree { token_trees, delimiter: template.delimiter }) 383 Ok(tt::Subtree { token_trees, delimiter: template.delimiter })
@@ -356,14 +412,23 @@ fn expand_tt(
356 let mut has_seps = 0; 412 let mut has_seps = 0;
357 let mut counter = 0; 413 let mut counter = 0;
358 414
415 // We store the old var expanded value, and restore it later
416 // It is because before this `$repeat`,
417 // it is possible some variables already expanad in the same subtree
418 //
419 // `some_var_expanded` keep check if the deeper subtree has expanded variables
359 let mut some_var_expanded = false; 420 let mut some_var_expanded = false;
421 let old_var_expanded = ctx.var_expanded;
360 ctx.var_expanded = false; 422 ctx.var_expanded = false;
361 423
362 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { 424 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
363 // if no var expaned in the child, we count it as a fail 425 // if no var expanded in the child, we count it as a fail
364 if !ctx.var_expanded { 426 if !ctx.var_expanded {
365 break; 427 break;
366 } 428 }
429
430 // Reset `ctx.var_expandeded` to see if there is other expanded variable
431 // in the next matching
367 some_var_expanded = true; 432 some_var_expanded = true;
368 ctx.var_expanded = false; 433 ctx.var_expanded = false;
369 434
@@ -407,7 +472,8 @@ fn expand_tt(
407 } 472 }
408 } 473 }
409 474
410 ctx.var_expanded = some_var_expanded; 475 // Restore the `var_expanded` by combining old one and the new one
476 ctx.var_expanded = some_var_expanded || old_var_expanded;
411 477
412 ctx.nesting.pop().unwrap(); 478 ctx.nesting.pop().unwrap();
413 for _ in 0..has_seps { 479 for _ in 0..has_seps {
@@ -433,6 +499,33 @@ fn expand_tt(
433 // FIXME: Properly handle $crate token 499 // FIXME: Properly handle $crate token
434 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() }) 500 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
435 .into() 501 .into()
502 } else if !ctx.bindings.contains(&v.text) {
503 // Note that it is possible to have a `$var` inside a macro which is not bound.
504 // For example:
505 // ```
506 // macro_rules! foo {
507 // ($a:ident, $b:ident, $c:tt) => {
508 // macro_rules! bar {
509 // ($bi:ident) => {
510 // fn $bi() -> u8 {$c}
511 // }
512 // }
513 // }
514 // ```
515 // We just treat it a normal tokens
516 tt::Subtree {
517 delimiter: tt::Delimiter::None,
518 token_trees: vec![
519 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
520 .into(),
521 tt::Leaf::from(tt::Ident {
522 text: v.text.clone(),
523 id: TokenId::unspecified(),
524 })
525 .into(),
526 ],
527 }
528 .into()
436 } else { 529 } else {
437 let tkn = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); 530 let tkn = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
438 ctx.var_expanded = true; 531 ctx.var_expanded = true;
@@ -459,11 +552,12 @@ mod tests {
459 552
460 #[test] 553 #[test]
461 fn test_expand_rule() { 554 fn test_expand_rule() {
462 assert_err( 555 // FIXME: The missing $var check should be in parsing phase
463 "($i:ident) => ($j)", 556 // assert_err(
464 "foo!{a}", 557 // "($i:ident) => ($j)",
465 ExpandError::BindingError(String::from("could not find binding `j`")), 558 // "foo!{a}",
466 ); 559 // ExpandError::BindingError(String::from("could not find binding `j`")),
560 // );
467 561
468 assert_err( 562 assert_err(
469 "($($i:ident);*) => ($i)", 563 "($($i:ident);*) => ($i)",
diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs
index c7ab463e2..797c70bc7 100644
--- a/crates/ra_mbe/src/mbe_parser.rs
+++ b/crates/ra_mbe/src/mbe_parser.rs
@@ -28,17 +28,31 @@ fn parse_rule(p: &mut TtCursor) -> Result<crate::Rule, ParseError> {
28 Ok(crate::Rule { lhs, rhs }) 28 Ok(crate::Rule { lhs, rhs })
29} 29}
30 30
31fn is_boolean_literal(lit: Option<&tt::TokenTree>) -> bool {
32 if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = lit {
33 if lit.text == "true" || lit.text == "false" {
34 return true;
35 }
36 }
37
38 false
39}
40
31fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree, ParseError> { 41fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree, ParseError> {
32 let mut token_trees = Vec::new(); 42 let mut token_trees = Vec::new();
33 let mut p = TtCursor::new(tt); 43 let mut p = TtCursor::new(tt);
34 while let Some(tt) = p.eat() { 44 while let Some(tt) = p.eat() {
35 let child: crate::TokenTree = match tt { 45 let child: crate::TokenTree = match tt {
36 tt::TokenTree::Leaf(leaf) => match leaf { 46 tt::TokenTree::Leaf(leaf) => match leaf {
37 tt::Leaf::Punct(tt::Punct { char: '$', .. }) => { 47 tt::Leaf::Punct(tt::Punct { char: '$', spacing }) => {
38 if p.at_ident().is_some() { 48 // mbe var can be an ident or keyword, including `true` and `false`
49 if p.at_ident().is_some() || is_boolean_literal(p.current()) {
39 crate::Leaf::from(parse_var(&mut p, transcriber)?).into() 50 crate::Leaf::from(parse_var(&mut p, transcriber)?).into()
40 } else { 51 } else if let Some(tt::TokenTree::Subtree(_)) = p.current() {
41 parse_repeat(&mut p, transcriber)?.into() 52 parse_repeat(&mut p, transcriber)?.into()
53 } else {
54 // Treat it as normal punct
55 crate::Leaf::from(tt::Punct { char: '$', spacing: *spacing }).into()
42 } 56 }
43 } 57 }
44 tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(), 58 tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(),
@@ -57,8 +71,16 @@ fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree,
57} 71}
58 72
59fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseError> { 73fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseError> {
60 let ident = p.eat_ident().unwrap(); 74 let text = {
61 let text = ident.text.clone(); 75 if is_boolean_literal(p.current()) {
76 let lit = p.eat_literal().unwrap();
77 lit.text.clone()
78 } else {
79 let ident = p.eat_ident().unwrap();
80 ident.text.clone()
81 }
82 };
83
62 let kind = if !transcriber && p.at_char(':') { 84 let kind = if !transcriber && p.at_char(':') {
63 p.bump(); 85 p.bump();
64 if let Some(ident) = p.eat_ident() { 86 if let Some(ident) = p.eat_ident() {
@@ -89,7 +111,7 @@ fn mk_repeat(
89} 111}
90 112
91fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> { 113fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> {
92 let subtree = p.eat_subtree().unwrap(); 114 let subtree = p.eat_subtree()?;
93 let mut subtree = parse_subtree(subtree, transcriber)?; 115 let mut subtree = parse_subtree(subtree, transcriber)?;
94 subtree.delimiter = crate::Delimiter::None; 116 subtree.delimiter = crate::Delimiter::None;
95 117
@@ -121,6 +143,10 @@ mod tests {
121 expect_err("invalid", "subtree"); 143 expect_err("invalid", "subtree");
122 144
123 is_valid("($i:ident) => ()"); 145 is_valid("($i:ident) => ()");
146 is_valid("($($i:ident)*) => ($_)");
147 is_valid("($($true:ident)*) => ($true)");
148 is_valid("($($false:ident)*) => ($false)");
149
124 expect_err("$i:ident => ()", "subtree"); 150 expect_err("$i:ident => ()", "subtree");
125 expect_err("($i:ident) ()", "`=`"); 151 expect_err("($i:ident) ()", "`=`");
126 expect_err("($($i:ident)_) => ()", "repeat"); 152 expect_err("($($i:ident)_) => ()", "repeat");
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 278d046fb..3554dc110 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -45,20 +45,6 @@ impl<'a> TokenSeq<'a> {
45 } 45 }
46 } 46 }
47 } 47 }
48
49 fn len(&self) -> usize {
50 match self {
51 TokenSeq::Subtree(subtree) => subtree.token_trees.len() + 2,
52 TokenSeq::Seq(tokens) => tokens.len(),
53 }
54 }
55
56 fn child_slice(&self, pos: usize) -> &[tt::TokenTree] {
57 match self {
58 TokenSeq::Subtree(subtree) => &subtree.token_trees[pos - 1..],
59 TokenSeq::Seq(tokens) => &tokens[pos..],
60 }
61 }
62} 48}
63 49
64#[derive(Debug, Clone, Eq, PartialEq)] 50#[derive(Debug, Clone, Eq, PartialEq)]
@@ -66,7 +52,6 @@ struct TtToken {
66 pub kind: SyntaxKind, 52 pub kind: SyntaxKind,
67 pub is_joint_to_next: bool, 53 pub is_joint_to_next: bool,
68 pub text: SmolStr, 54 pub text: SmolStr,
69 pub n_tokens: usize,
70} 55}
71 56
72#[derive(Debug, Clone, Eq, PartialEq)] 57#[derive(Debug, Clone, Eq, PartialEq)]
@@ -80,19 +65,12 @@ struct SubTreeWalker<'a> {
80 pos: usize, 65 pos: usize,
81 stack: Vec<(TokenSeq<'a>, usize)>, 66 stack: Vec<(TokenSeq<'a>, usize)>,
82 cursor: WalkCursor, 67 cursor: WalkCursor,
83 last_steps: Vec<usize>,
84 ts: TokenSeq<'a>, 68 ts: TokenSeq<'a>,
85} 69}
86 70
87impl<'a> SubTreeWalker<'a> { 71impl<'a> SubTreeWalker<'a> {
88 fn new(ts: TokenSeq<'a>) -> SubTreeWalker { 72 fn new(ts: TokenSeq<'a>) -> SubTreeWalker {
89 let mut res = SubTreeWalker { 73 let mut res = SubTreeWalker { pos: 0, stack: vec![], cursor: WalkCursor::Eof, ts };
90 pos: 0,
91 stack: vec![],
92 cursor: WalkCursor::Eof,
93 last_steps: vec![],
94 ts,
95 };
96 74
97 res.reset(); 75 res.reset();
98 res 76 res
@@ -105,7 +83,6 @@ impl<'a> SubTreeWalker<'a> {
105 fn reset(&mut self) { 83 fn reset(&mut self) {
106 self.pos = 0; 84 self.pos = 0;
107 self.stack = vec![]; 85 self.stack = vec![];
108 self.last_steps = vec![];
109 86
110 self.cursor = match self.ts.get(0) { 87 self.cursor = match self.ts.get(0) {
111 DelimToken::Token(token) => match token { 88 DelimToken::Token(token) => match token {
@@ -114,10 +91,7 @@ impl<'a> SubTreeWalker<'a> {
114 self.stack.push((ts, 0)); 91 self.stack.push((ts, 0));
115 WalkCursor::Token(0, convert_delim(subtree.delimiter, false)) 92 WalkCursor::Token(0, convert_delim(subtree.delimiter, false))
116 } 93 }
117 tt::TokenTree::Leaf(leaf) => { 94 tt::TokenTree::Leaf(leaf) => WalkCursor::Token(0, convert_leaf(leaf)),
118 let next_tokens = self.ts.child_slice(0);
119 WalkCursor::Token(0, convert_leaf(&next_tokens, leaf))
120 }
121 }, 95 },
122 DelimToken::Delim(delim, is_end) => { 96 DelimToken::Delim(delim, is_end) => {
123 assert!(!is_end); 97 assert!(!is_end);
@@ -138,24 +112,6 @@ impl<'a> SubTreeWalker<'a> {
138 self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts) 112 self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts)
139 } 113 }
140 114
141 /// Move cursor backward by 1 step
142 fn backward(&mut self) {
143 if self.last_steps.is_empty() {
144 return;
145 }
146
147 self.pos -= 1;
148 let last_step = self.last_steps.pop().unwrap();
149
150 self.cursor = match self.cursor {
151 WalkCursor::Token(idx, _) => self.walk_token(idx, last_step, true),
152 WalkCursor::Eof => {
153 let len = self.top().len();
154 self.walk_token(len, last_step, true)
155 }
156 }
157 }
158
159 /// Move cursor forward by 1 step 115 /// Move cursor forward by 1 step
160 fn forward(&mut self) { 116 fn forward(&mut self) {
161 if self.is_eof() { 117 if self.is_eof() {
@@ -163,37 +119,24 @@ impl<'a> SubTreeWalker<'a> {
163 } 119 }
164 self.pos += 1; 120 self.pos += 1;
165 121
166 let step = self.current().map(|x| x.n_tokens).unwrap_or(1);
167 self.last_steps.push(step);
168
169 if let WalkCursor::Token(u, _) = self.cursor { 122 if let WalkCursor::Token(u, _) = self.cursor {
170 self.cursor = self.walk_token(u, step, false) 123 self.cursor = self.walk_token(u)
171 } 124 }
172 } 125 }
173 126
174 /// Traversal child token 127 /// Traversal child token
175 fn walk_token(&mut self, pos: usize, offset: usize, backward: bool) -> WalkCursor { 128 fn walk_token(&mut self, pos: usize) -> WalkCursor {
176 let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts); 129 let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts);
177 130 let pos = pos + 1;
178 if backward && pos < offset {
179 let (_, last_idx) = self.stack.pop().unwrap();
180 return self.walk_token(last_idx, offset, backward);
181 }
182
183 let pos = if backward { pos - offset } else { pos + offset };
184 131
185 match top.get(pos) { 132 match top.get(pos) {
186 DelimToken::Token(token) => match token { 133 DelimToken::Token(token) => match token {
187 tt::TokenTree::Subtree(subtree) => { 134 tt::TokenTree::Subtree(subtree) => {
188 let ts = TokenSeq::from(subtree); 135 let ts = TokenSeq::from(subtree);
189 let new_idx = if backward { ts.len() - 1 } else { 0 };
190 self.stack.push((ts, pos)); 136 self.stack.push((ts, pos));
191 WalkCursor::Token(new_idx, convert_delim(subtree.delimiter, backward)) 137 WalkCursor::Token(0, convert_delim(subtree.delimiter, false))
192 }
193 tt::TokenTree::Leaf(leaf) => {
194 let next_tokens = top.child_slice(pos);
195 WalkCursor::Token(pos, convert_leaf(&next_tokens, leaf))
196 } 138 }
139 tt::TokenTree::Leaf(leaf) => WalkCursor::Token(pos, convert_leaf(leaf)),
197 }, 140 },
198 DelimToken::Delim(delim, is_end) => { 141 DelimToken::Delim(delim, is_end) => {
199 WalkCursor::Token(pos, convert_delim(*delim, is_end)) 142 WalkCursor::Token(pos, convert_delim(*delim, is_end))
@@ -201,8 +144,7 @@ impl<'a> SubTreeWalker<'a> {
201 DelimToken::End => { 144 DelimToken::End => {
202 // it is the top level 145 // it is the top level
203 if let Some((_, last_idx)) = self.stack.pop() { 146 if let Some((_, last_idx)) = self.stack.pop() {
204 assert!(!backward); 147 self.walk_token(last_idx)
205 self.walk_token(last_idx, offset, backward)
206 } else { 148 } else {
207 WalkCursor::Eof 149 WalkCursor::Eof
208 } 150 }
@@ -237,12 +179,9 @@ impl<'a> WalkerOwner<'a> {
237 } 179 }
238 180
239 while pos >= cached.len() { 181 while pos >= cached.len() {
240 let len = cached.len(); 182 self.set_pos(cached.len());
241 cached.push({ 183 let walker = self.walker.borrow();
242 self.set_pos(len); 184 cached.push(walker.current().cloned());
243 let walker = self.walker.borrow();
244 walker.current().cloned()
245 });
246 } 185 }
247 186
248 return cached[pos].clone(); 187 return cached[pos].clone();
@@ -250,12 +189,11 @@ impl<'a> WalkerOwner<'a> {
250 189
251 fn set_pos(&self, pos: usize) { 190 fn set_pos(&self, pos: usize) {
252 let mut walker = self.walker.borrow_mut(); 191 let mut walker = self.walker.borrow_mut();
192 assert!(walker.pos <= pos);
193
253 while pos > walker.pos && !walker.is_eof() { 194 while pos > walker.pos && !walker.is_eof() {
254 walker.forward(); 195 walker.forward();
255 } 196 }
256 while pos < walker.pos {
257 walker.backward();
258 }
259 } 197 }
260 198
261 fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { 199 fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> {
@@ -264,15 +202,16 @@ impl<'a> WalkerOwner<'a> {
264 walker.reset(); 202 walker.reset();
265 203
266 while walker.pos < n { 204 while walker.pos < n {
267 if let WalkCursor::Token(u, tt) = &walker.cursor { 205 if let WalkCursor::Token(u, _) = &walker.cursor {
268 // We only collect the topmost child 206 // We only collect the topmost child
269 if walker.stack.len() == 0 { 207 if walker.stack.len() == 0 {
270 for i in 0..tt.n_tokens { 208 if let DelimToken::Token(token) = walker.ts.get(*u) {
271 if let DelimToken::Token(token) = walker.ts.get(u + i) { 209 res.push(token);
272 res.push(token);
273 }
274 } 210 }
275 } else if walker.stack.len() == 1 { 211 }
212 // Check whether the second level is a subtree
213 // if so, collect its parent which is topmost child
214 else if walker.stack.len() == 1 {
276 if let DelimToken::Delim(_, is_end) = walker.top().get(*u) { 215 if let DelimToken::Delim(_, is_end) = walker.top().get(*u) {
277 if !is_end { 216 if !is_end {
278 let (_, last_idx) = &walker.stack[0]; 217 let (_, last_idx) = &walker.stack[0];
@@ -343,78 +282,6 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
343 } 282 }
344} 283}
345 284
346pub(crate) struct TokenPeek<'a, I>
347where
348 I: Iterator<Item = &'a tt::TokenTree>,
349{
350 iter: itertools::MultiPeek<I>,
351}
352
353// helper function
354fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
355 if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
356 return Some(pp);
357 }
358 None
359}
360
361impl<'a, I> TokenPeek<'a, I>
362where
363 I: Iterator<Item = &'a tt::TokenTree>,
364{
365 pub fn new(iter: I) -> Self {
366 TokenPeek { iter: itertools::multipeek(iter) }
367 }
368
369 pub fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> {
370 if p.spacing != tt::Spacing::Joint {
371 return None;
372 }
373
374 self.iter.reset_peek();
375 let p1 = to_punct(self.iter.peek()?)?;
376 Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
377 }
378
379 pub fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> {
380 self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
381 if !last_joint {
382 None
383 } else {
384 let p2 = to_punct(*self.iter.peek()?)?;
385 Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
386 }
387 })
388 }
389}
390
391// FIXME: Remove this function
392fn convert_multi_char_punct<'b, I>(
393 p: &tt::Punct,
394 iter: &mut TokenPeek<'b, I>,
395) -> Option<(SyntaxKind, bool, &'static str, usize)>
396where
397 I: Iterator<Item = &'b tt::TokenTree>,
398{
399 if let Some((m, is_joint_to_next)) = iter.current_punct3(p) {
400 if let Some((kind, text)) = match m {
401 _ => None,
402 } {
403 return Some((kind, is_joint_to_next, text, 3));
404 }
405 }
406
407 if let Some((m, is_joint_to_next)) = iter.current_punct2(p) {
408 if let Some((kind, text)) = match m {
409 _ => None,
410 } {
411 return Some((kind, is_joint_to_next, text, 2));
412 }
413 }
414
415 None
416}
417
418fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken { 285fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken {
419 let (kinds, texts) = match d { 286 let (kinds, texts) = match d {
420 tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), 287 tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"),
@@ -426,7 +293,7 @@ fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken {
426 let idx = closing as usize; 293 let idx = closing as usize;
427 let kind = kinds[idx]; 294 let kind = kinds[idx];
428 let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" }; 295 let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" };
429 TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 } 296 TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
430} 297}
431 298
432fn convert_literal(l: &tt::Literal) -> TtToken { 299fn convert_literal(l: &tt::Literal) -> TtToken {
@@ -437,7 +304,7 @@ fn convert_literal(l: &tt::Literal) -> TtToken {
437 _ => panic!("Fail to convert given literal {:#?}", &l), 304 _ => panic!("Fail to convert given literal {:#?}", &l),
438 }); 305 });
439 306
440 TtToken { kind, is_joint_to_next: false, text: l.text.clone(), n_tokens: 1 } 307 TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
441} 308}
442 309
443fn convert_ident(ident: &tt::Ident) -> TtToken { 310fn convert_ident(ident: &tt::Ident) -> TtToken {
@@ -447,39 +314,31 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
447 SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT) 314 SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT)
448 }; 315 };
449 316
450 TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } 317 TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
451} 318}
452 319
453fn convert_punct(p: &tt::Punct, next_tokens: &[tt::TokenTree]) -> TtToken { 320fn convert_punct(p: &tt::Punct) -> TtToken {
454 let mut iter = next_tokens.iter(); 321 let kind = match p.char {
455 iter.next(); 322 // lexer may produce compound tokens for these ones
456 let mut peek = TokenPeek::new(iter); 323 '.' => DOT,
457 324 ':' => COLON,
458 if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { 325 '=' => EQ,
459 TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size } 326 '!' => EXCL,
460 } else { 327 '-' => MINUS,
461 let kind = match p.char { 328 c => SyntaxKind::from_char(c).unwrap(),
462 // lexer may produce combpund tokens for these ones 329 };
463 '.' => DOT, 330 let text = {
464 ':' => COLON, 331 let mut buf = [0u8; 4];
465 '=' => EQ, 332 let s: &str = p.char.encode_utf8(&mut buf);
466 '!' => EXCL, 333 SmolStr::new(s)
467 '-' => MINUS, 334 };
468 c => SyntaxKind::from_char(c).unwrap(), 335 TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
469 };
470 let text = {
471 let mut buf = [0u8; 4];
472 let s: &str = p.char.encode_utf8(&mut buf);
473 SmolStr::new(s)
474 };
475 TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 }
476 }
477} 336}
478 337
479fn convert_leaf(tokens: &[tt::TokenTree], leaf: &tt::Leaf) -> TtToken { 338fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
480 match leaf { 339 match leaf {
481 tt::Leaf::Literal(l) => convert_literal(l), 340 tt::Leaf::Literal(l) => convert_literal(l),
482 tt::Leaf::Ident(ident) => convert_ident(ident), 341 tt::Leaf::Ident(ident) => convert_ident(ident),
483 tt::Leaf::Punct(punct) => convert_punct(punct, tokens), 342 tt::Leaf::Punct(punct) => convert_punct(punct),
484 } 343 }
485} 344}
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index e0f228ce9..73a0780da 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -118,6 +118,69 @@ impl TokenMap {
118 } 118 }
119} 119}
120 120
121/// Returns the textual content of a doc comment block as a quoted string
122/// That is, strips leading `///` (or `/**`, etc)
123/// and strips the ending `*/`
124/// And then quote the string, which is needed to convert to `tt::Literal`
125fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
126 use ast::AstToken;
127
128 let prefix_len = comment.prefix().len();
129 let mut text = &comment.text()[prefix_len..];
130
131 // Remove ending "*/"
132 if comment.kind().shape == ast::CommentShape::Block {
133 text = &text[0..text.len() - 2];
134 }
135
136 // Quote the string
137 // Note that `tt::Literal` expect an escaped string
138 let text = format!("{:?}", text.escape_default().to_string());
139 text.into()
140}
141
142fn convert_doc_comment<'a>(token: &ra_syntax::SyntaxToken<'a>) -> Option<Vec<tt::TokenTree>> {
143 use ast::AstToken;
144 let comment = ast::Comment::cast(*token)?;
145 let doc = comment.kind().doc?;
146
147 // Make `doc="\" Comments\""
148 let mut meta_tkns = Vec::new();
149 meta_tkns.push(mk_ident("doc"));
150 meta_tkns.push(mk_punct('='));
151 meta_tkns.push(mk_doc_literal(&comment));
152
153 // Make `#![]`
154 let mut token_trees = Vec::new();
155 token_trees.push(mk_punct('#'));
156 if let ast::CommentPlacement::Inner = doc {
157 token_trees.push(mk_punct('!'));
158 }
159 token_trees.push(tt::TokenTree::from(tt::Subtree::from(
160 tt::Subtree { delimiter: tt::Delimiter::Bracket, token_trees: meta_tkns }.into(),
161 )));
162
163 return Some(token_trees);
164
165 // Helper functions
166 fn mk_ident(s: &str) -> tt::TokenTree {
167 tt::TokenTree::from(tt::Leaf::from(tt::Ident {
168 text: s.into(),
169 id: tt::TokenId::unspecified(),
170 }))
171 }
172
173 fn mk_punct(c: char) -> tt::TokenTree {
174 tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone }))
175 }
176
177 fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
178 let lit = tt::Literal { text: doc_comment_text(comment) };
179
180 tt::TokenTree::from(tt::Leaf::from(lit))
181 }
182}
183
121fn convert_tt( 184fn convert_tt(
122 token_map: &mut TokenMap, 185 token_map: &mut TokenMap,
123 global_offset: TextUnit, 186 global_offset: TextUnit,
@@ -141,37 +204,32 @@ fn convert_tt(
141 let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); 204 let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
142 205
143 while let Some(child) = child_iter.next() { 206 while let Some(child) = child_iter.next() {
144 if (skip_first && (child == first_child || child == last_child)) || child.kind().is_trivia() 207 if skip_first && (child == first_child || child == last_child) {
145 {
146 continue; 208 continue;
147 } 209 }
210
148 match child { 211 match child {
149 SyntaxElement::Token(token) => { 212 SyntaxElement::Token(token) => {
150 if token.kind().is_punct() { 213 if let Some(doc_tokens) = convert_doc_comment(&token) {
151 let mut prev = None; 214 token_trees.extend(doc_tokens);
152 for char in token.text().chars() { 215 } else if token.kind().is_trivia() {
153 if let Some(char) = prev { 216 continue;
154 token_trees.push( 217 } else if token.kind().is_punct() {
155 tt::Leaf::from(tt::Punct { char, spacing: tt::Spacing::Joint }) 218 assert!(token.text().len() == 1, "Input ast::token punct must be single char.");
156 .into(), 219 let char = token.text().chars().next().unwrap();
157 ); 220
158 } 221 let spacing = match child_iter.peek() {
159 prev = Some(char) 222 Some(SyntaxElement::Token(token)) => {
160 } 223 if token.kind().is_punct() {
161 if let Some(char) = prev { 224 tt::Spacing::Joint
162 let spacing = match child_iter.peek() { 225 } else {
163 Some(SyntaxElement::Token(token)) => { 226 tt::Spacing::Alone
164 if token.kind().is_punct() {
165 tt::Spacing::Joint
166 } else {
167 tt::Spacing::Alone
168 }
169 } 227 }
170 _ => tt::Spacing::Alone, 228 }
171 }; 229 _ => tt::Spacing::Alone,
230 };
172 231
173 token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); 232 token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into());
174 }
175 } else { 233 } else {
176 let child: tt::TokenTree = if token.kind() == SyntaxKind::TRUE_KW 234 let child: tt::TokenTree = if token.kind() == SyntaxKind::TRUE_KW
177 || token.kind() == SyntaxKind::FALSE_KW 235 || token.kind() == SyntaxKind::FALSE_KW
@@ -224,6 +282,15 @@ impl<'a, Q: Querier> TtTreeSink<'a, Q> {
224 } 282 }
225} 283}
226 284
285fn is_delimiter(kind: SyntaxKind) -> bool {
286 use SyntaxKind::*;
287
288 match kind {
289 L_PAREN | L_BRACK | L_CURLY | R_PAREN | R_BRACK | R_CURLY => true,
290 _ => false,
291 }
292}
293
227impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { 294impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> {
228 fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { 295 fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
229 if kind == L_DOLLAR || kind == R_DOLLAR { 296 if kind == L_DOLLAR || kind == R_DOLLAR {
@@ -240,14 +307,18 @@ impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> {
240 self.buf.clear(); 307 self.buf.clear();
241 self.inner.token(kind, text); 308 self.inner.token(kind, text);
242 309
243 // // Add a white space to token 310 // Add a white space between tokens, only if both are not delimiters
244 // let (last_kind, _, last_joint_to_next ) = self.src_querier.token(self.token_pos-n_tokens as usize); 311 if !is_delimiter(kind) {
245 // if !last_joint_to_next && last_kind.is_punct() { 312 let (last_kind, _, last_joint_to_next) = self.src_querier.token(self.token_pos - 1);
246 // let (cur_kind, _, _ ) = self.src_querier.token(self.token_pos); 313 if !last_joint_to_next && last_kind.is_punct() {
247 // if cur_kind.is_punct() { 314 let (cur_kind, _, _) = self.src_querier.token(self.token_pos);
248 // self.inner.token(WHITESPACE, " ".into()); 315 if !is_delimiter(cur_kind) {
249 // } 316 if cur_kind.is_punct() {
250 // } 317 self.inner.token(WHITESPACE, " ".into());
318 }
319 }
320 }
321 }
251 } 322 }
252 323
253 fn start_node(&mut self, kind: SyntaxKind) { 324 fn start_node(&mut self, kind: SyntaxKind) {
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
new file mode 100644
index 000000000..c487bbbd4
--- /dev/null
+++ b/crates/ra_mbe/src/tests.rs
@@ -0,0 +1,1399 @@
1use ra_syntax::{ast, AstNode};
2
3use super::*;
4
5// Good first issue (although a slightly challenging one):
6//
7// * Pick a random test from here
8// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
9// * Port the test to rust and add it to this module
10// * Make it pass :-)
11
12#[test]
13fn test_convert_tt() {
14 let macro_definition = r#"
15macro_rules! impl_froms {
16 ($e:ident: $($v:ident),*) => {
17 $(
18 impl From<$v> for $e {
19 fn from(it: $v) -> $e {
20 $e::$v(it)
21 }
22 }
23 )*
24 }
25}
26"#;
27
28 let macro_invocation = r#"
29impl_froms!(TokenTree: Leaf, Subtree);
30"#;
31
32 let source_file = ast::SourceFile::parse(macro_definition);
33 let macro_definition =
34 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
35
36 let source_file = ast::SourceFile::parse(macro_invocation);
37 let macro_invocation =
38 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
39
40 let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
41 let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
42 let rules = crate::MacroRules::parse(&definition_tt).unwrap();
43 let expansion = rules.expand(&invocation_tt).unwrap();
44 assert_eq!(
45 expansion.to_string(),
46 "impl From <Leaf > for TokenTree {fn from (it : Leaf) -> TokenTree {TokenTree ::Leaf (it)}} \
47 impl From <Subtree > for TokenTree {fn from (it : Subtree) -> TokenTree {TokenTree ::Subtree (it)}}"
48 )
49}
50
51pub(crate) fn create_rules(macro_definition: &str) -> MacroRules {
52 let source_file = ast::SourceFile::parse(macro_definition);
53 let macro_definition =
54 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
55
56 let (definition_tt, _) = ast_to_token_tree(macro_definition.token_tree().unwrap()).unwrap();
57 crate::MacroRules::parse(&definition_tt).unwrap()
58}
59
60pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
61 let source_file = ast::SourceFile::parse(invocation);
62 let macro_invocation =
63 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
64
65 let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
66
67 rules.expand(&invocation_tt).unwrap()
68}
69
70pub(crate) fn expand_to_items(
71 rules: &MacroRules,
72 invocation: &str,
73) -> ra_syntax::TreeArc<ast::MacroItems> {
74 let expanded = expand(rules, invocation);
75 token_tree_to_macro_items(&expanded).unwrap()
76}
77
78#[allow(unused)]
79pub(crate) fn expand_to_stmts(
80 rules: &MacroRules,
81 invocation: &str,
82) -> ra_syntax::TreeArc<ast::MacroStmts> {
83 let expanded = expand(rules, invocation);
84 token_tree_to_macro_stmts(&expanded).unwrap()
85}
86
87pub(crate) fn expand_to_expr(
88 rules: &MacroRules,
89 invocation: &str,
90) -> ra_syntax::TreeArc<ast::Expr> {
91 let expanded = expand(rules, invocation);
92 token_tree_to_expr(&expanded).unwrap()
93}
94
95pub(crate) fn text_to_tokentree(text: &str) -> tt::Subtree {
96 // wrap the given text to a macro call
97 let wrapped = format!("wrap_macro!( {} )", text);
98 let wrapped = ast::SourceFile::parse(&wrapped);
99 let wrapped = wrapped.syntax().descendants().find_map(ast::TokenTree::cast).unwrap();
100 let mut wrapped = ast_to_token_tree(wrapped).unwrap().0;
101 wrapped.delimiter = tt::Delimiter::None;
102
103 wrapped
104}
105
106pub(crate) enum MacroKind {
107 Items,
108 Stmts,
109}
110
111use ra_syntax::WalkEvent;
112
113pub fn debug_dump_ignore_spaces(node: &ra_syntax::SyntaxNode) -> String {
114 use std::fmt::Write;
115
116 let mut level = 0;
117 let mut buf = String::new();
118 macro_rules! indent {
119 () => {
120 for _ in 0..level {
121 buf.push_str(" ");
122 }
123 };
124 }
125
126 for event in node.preorder_with_tokens() {
127 match event {
128 WalkEvent::Enter(element) => {
129 match element {
130 ra_syntax::SyntaxElement::Node(node) => {
131 indent!();
132 writeln!(buf, "{:?}", node.kind()).unwrap();
133 }
134 ra_syntax::SyntaxElement::Token(token) => match token.kind() {
135 ra_syntax::SyntaxKind::WHITESPACE => {}
136 _ => {
137 indent!();
138 writeln!(buf, "{:?}", token.kind()).unwrap();
139 }
140 },
141 }
142 level += 1;
143 }
144 WalkEvent::Leave(_) => level -= 1,
145 }
146 }
147
148 buf
149}
150
151pub(crate) fn assert_expansion(
152 kind: MacroKind,
153 rules: &MacroRules,
154 invocation: &str,
155 expected: &str,
156) -> tt::Subtree {
157 let expanded = expand(rules, invocation);
158 assert_eq!(expanded.to_string(), expected);
159
160 let expected = expected.replace("$crate", "C_C__C");
161
162 // wrap the given text to a macro call
163 let expected = text_to_tokentree(&expected);
164
165 let (expanded_tree, expected_tree) = match kind {
166 MacroKind::Items => {
167 let expanded_tree = token_tree_to_macro_items(&expanded);
168 let expected_tree = token_tree_to_macro_items(&expected);
169
170 (
171 debug_dump_ignore_spaces(expanded_tree.unwrap().syntax()).trim().to_string(),
172 debug_dump_ignore_spaces(expected_tree.unwrap().syntax()).trim().to_string(),
173 )
174 }
175
176 MacroKind::Stmts => {
177 let expanded_tree = token_tree_to_macro_stmts(&expanded);
178 let expected_tree = token_tree_to_macro_stmts(&expected);
179
180 (
181 debug_dump_ignore_spaces(expanded_tree.unwrap().syntax()).trim().to_string(),
182 debug_dump_ignore_spaces(expected_tree.unwrap().syntax()).trim().to_string(),
183 )
184 }
185 };
186
187 let expected_tree = expected_tree.replace("C_C__C", "$crate");
188 assert_eq!(
189 expanded_tree, expected_tree,
190 "left => {}\nright => {}",
191 expanded_tree, expected_tree,
192 );
193
194 expanded
195}
196
197#[test]
198fn test_fail_match_pattern_by_first_token() {
199 let rules = create_rules(
200 r#"
201 macro_rules! foo {
202 ($ i:ident) => (
203 mod $ i {}
204 );
205 (= $ i:ident) => (
206 fn $ i() {}
207 );
208 (+ $ i:ident) => (
209 struct $ i;
210 )
211 }
212"#,
213 );
214
215 assert_expansion(MacroKind::Items, &rules, "foo! { foo }", "mod foo {}");
216 assert_expansion(MacroKind::Items, &rules, "foo! { = bar }", "fn bar () {}");
217 assert_expansion(MacroKind::Items, &rules, "foo! { + Baz }", "struct Baz ;");
218}
219
220#[test]
221fn test_fail_match_pattern_by_last_token() {
222 let rules = create_rules(
223 r#"
224 macro_rules! foo {
225 ($ i:ident) => (
226 mod $ i {}
227 );
228 ($ i:ident =) => (
229 fn $ i() {}
230 );
231 ($ i:ident +) => (
232 struct $ i;
233 )
234 }
235"#,
236 );
237
238 assert_expansion(MacroKind::Items, &rules, "foo! { foo }", "mod foo {}");
239 assert_expansion(MacroKind::Items, &rules, "foo! { bar = }", "fn bar () {}");
240 assert_expansion(MacroKind::Items, &rules, "foo! { Baz + }", "struct Baz ;");
241}
242
243#[test]
244fn test_fail_match_pattern_by_word_token() {
245 let rules = create_rules(
246 r#"
247 macro_rules! foo {
248 ($ i:ident) => (
249 mod $ i {}
250 );
251 (spam $ i:ident) => (
252 fn $ i() {}
253 );
254 (eggs $ i:ident) => (
255 struct $ i;
256 )
257 }
258"#,
259 );
260
261 assert_expansion(MacroKind::Items, &rules, "foo! { foo }", "mod foo {}");
262 assert_expansion(MacroKind::Items, &rules, "foo! { spam bar }", "fn bar () {}");
263 assert_expansion(MacroKind::Items, &rules, "foo! { eggs Baz }", "struct Baz ;");
264}
265
266#[test]
267fn test_match_group_pattern_by_separator_token() {
268 let rules = create_rules(
269 r#"
270 macro_rules! foo {
271 ($ ($ i:ident),*) => ($ (
272 mod $ i {}
273 )*);
274 ($ ($ i:ident)#*) => ($ (
275 fn $ i() {}
276 )*);
277 ($ i:ident ,# $ j:ident) => (
278 struct $ i;
279 struct $ j;
280 )
281 }
282"#,
283 );
284
285 assert_expansion(MacroKind::Items, &rules, "foo! { foo, bar }", "mod foo {} mod bar {}");
286 assert_expansion(MacroKind::Items, &rules, "foo! { foo# bar }", "fn foo () {} fn bar () {}");
287 assert_expansion(MacroKind::Items, &rules, "foo! { Foo,# Bar }", "struct Foo ; struct Bar ;");
288}
289
290#[test]
291fn test_match_group_pattern_with_multiple_defs() {
292 let rules = create_rules(
293 r#"
294 macro_rules! foo {
295 ($ ($ i:ident),*) => ( struct Bar { $ (
296 fn $ i {}
297 )*} );
298 }
299"#,
300 );
301
302 assert_expansion(
303 MacroKind::Items,
304 &rules,
305 "foo! { foo, bar }",
306 "struct Bar {fn foo {} fn bar {}}",
307 );
308}
309
310#[test]
311fn test_match_group_pattern_with_multiple_statement() {
312 let rules = create_rules(
313 r#"
314 macro_rules! foo {
315 ($ ($ i:ident),*) => ( fn baz { $ (
316 $ i ();
317 )*} );
318 }
319"#,
320 );
321
322 assert_expansion(MacroKind::Items, &rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}");
323}
324
325#[test]
326fn test_match_group_pattern_with_multiple_statement_without_semi() {
327 let rules = create_rules(
328 r#"
329 macro_rules! foo {
330 ($ ($ i:ident),*) => ( fn baz { $ (
331 $i()
332 );*} );
333 }
334"#,
335 );
336
337 assert_expansion(MacroKind::Items, &rules, "foo! { foo, bar }", "fn baz {foo () ;bar ()}");
338}
339
340#[test]
341fn test_match_group_empty_fixed_token() {
342 let rules = create_rules(
343 r#"
344 macro_rules! foo {
345 ($ ($ i:ident)* #abc) => ( fn baz { $ (
346 $ i ();
347 )*} );
348 }
349"#,
350 );
351
352 assert_expansion(MacroKind::Items, &rules, "foo! {#abc}", "fn baz {}");
353}
354
355#[test]
356fn test_match_group_in_subtree() {
357 let rules = create_rules(
358 r#"
359 macro_rules! foo {
360 (fn $name:ident {$($i:ident)*} ) => ( fn $name() { $ (
361 $ i ();
362 )*} );
363 }"#,
364 );
365
366 assert_expansion(MacroKind::Items, &rules, "foo! {fn baz {a b} }", "fn baz () {a () ; b () ;}");
367}
368
369#[test]
370fn test_match_group_with_multichar_sep() {
371 let rules = create_rules(
372 r#"
373 macro_rules! foo {
374 (fn $name:ident {$($i:literal)*} ) => ( fn $name() -> bool { $($i)&&*} );
375 }"#,
376 );
377
378 assert_expansion(
379 MacroKind::Items,
380 &rules,
381 "foo! (fn baz {true true} )",
382 "fn baz () -> bool {true &&true}",
383 );
384}
385
386#[test]
387fn test_match_group_zero_match() {
388 let rules = create_rules(
389 r#"
390 macro_rules! foo {
391 ( $($i:ident)* ) => ();
392 }"#,
393 );
394
395 assert_expansion(MacroKind::Items, &rules, "foo! ()", "");
396}
397
398#[test]
399fn test_match_group_in_group() {
400 let rules = create_rules(
401 r#"
402 macro_rules! foo {
403 { $( ( $($i:ident)* ) )* } => ( $( ( $($i)* ) )* );
404 }"#,
405 );
406
407 assert_expansion(MacroKind::Items, &rules, "foo! ( (a b) )", "(a b)");
408}
409
410#[test]
411fn test_expand_to_item_list() {
412 let rules = create_rules(
413 "
414 macro_rules! structs {
415 ($($i:ident),*) => {
416 $(struct $i { field: u32 } )*
417 }
418 }
419 ",
420 );
421 let expansion = expand(&rules, "structs!(Foo, Bar)");
422 let tree = token_tree_to_macro_items(&expansion);
423 assert_eq!(
424 tree.unwrap().syntax().debug_dump().trim(),
425 r#"
426MACRO_ITEMS@[0; 40)
427 STRUCT_DEF@[0; 20)
428 STRUCT_KW@[0; 6) "struct"
429 NAME@[6; 9)
430 IDENT@[6; 9) "Foo"
431 NAMED_FIELD_DEF_LIST@[9; 20)
432 L_CURLY@[9; 10) "{"
433 NAMED_FIELD_DEF@[10; 19)
434 NAME@[10; 15)
435 IDENT@[10; 15) "field"
436 COLON@[15; 16) ":"
437 PATH_TYPE@[16; 19)
438 PATH@[16; 19)
439 PATH_SEGMENT@[16; 19)
440 NAME_REF@[16; 19)
441 IDENT@[16; 19) "u32"
442 R_CURLY@[19; 20) "}"
443 STRUCT_DEF@[20; 40)
444 STRUCT_KW@[20; 26) "struct"
445 NAME@[26; 29)
446 IDENT@[26; 29) "Bar"
447 NAMED_FIELD_DEF_LIST@[29; 40)
448 L_CURLY@[29; 30) "{"
449 NAMED_FIELD_DEF@[30; 39)
450 NAME@[30; 35)
451 IDENT@[30; 35) "field"
452 COLON@[35; 36) ":"
453 PATH_TYPE@[36; 39)
454 PATH@[36; 39)
455 PATH_SEGMENT@[36; 39)
456 NAME_REF@[36; 39)
457 IDENT@[36; 39) "u32"
458 R_CURLY@[39; 40) "}""#
459 .trim()
460 );
461}
462
463#[test]
464fn test_expand_literals_to_token_tree() {
465 fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
466 if let tt::TokenTree::Subtree(subtree) = tt {
467 return &subtree;
468 }
469 unreachable!("It is not a subtree");
470 }
471
472 fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
473 if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
474 return lit;
475 }
476 unreachable!("It is not a literal");
477 }
478
479 let rules = create_rules(
480 r#"
481 macro_rules! literals {
482 ($i:ident) => {
483 {
484 let a = 'c';
485 let c = 1000;
486 let f = 12E+99_f64;
487 let s = "rust1";
488 }
489 }
490 }
491 "#,
492 );
493 let expansion = expand(&rules, "literals!(foo)");
494 let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees;
495
496 // [let] [a] [=] ['c'] [;]
497 assert_eq!(to_literal(&stm_tokens[3]).text, "'c'");
498 // [let] [c] [=] [1000] [;]
499 assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000");
500 // [let] [f] [=] [12E+99_f64] [;]
501 assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64");
502 // [let] [s] [=] ["rust1"] [;]
503 assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
504}
505
506#[test]
507fn test_two_idents() {
508 let rules = create_rules(
509 r#"
510 macro_rules! foo {
511 ($ i:ident, $ j:ident) => {
512 fn foo() { let a = $ i; let b = $j; }
513 }
514 }
515"#,
516 );
517 assert_expansion(
518 MacroKind::Items,
519 &rules,
520 "foo! { foo, bar }",
521 "fn foo () {let a = foo ; let b = bar ;}",
522 );
523}
524
525#[test]
526fn test_tt_to_stmts() {
527 let rules = create_rules(
528 r#"
529 macro_rules! foo {
530 () => {
531 let a = 0;
532 a = 10 + 1;
533 a
534 }
535 }
536"#,
537 );
538
539 let expanded = expand(&rules, "foo!{}");
540 let stmts = token_tree_to_macro_stmts(&expanded);
541
542 assert_eq!(
543 stmts.unwrap().syntax().debug_dump().trim(),
544 r#"MACRO_STMTS@[0; 15)
545 LET_STMT@[0; 7)
546 LET_KW@[0; 3) "let"
547 BIND_PAT@[3; 4)
548 NAME@[3; 4)
549 IDENT@[3; 4) "a"
550 EQ@[4; 5) "="
551 LITERAL@[5; 6)
552 INT_NUMBER@[5; 6) "0"
553 SEMI@[6; 7) ";"
554 EXPR_STMT@[7; 14)
555 BIN_EXPR@[7; 13)
556 PATH_EXPR@[7; 8)
557 PATH@[7; 8)
558 PATH_SEGMENT@[7; 8)
559 NAME_REF@[7; 8)
560 IDENT@[7; 8) "a"
561 EQ@[8; 9) "="
562 BIN_EXPR@[9; 13)
563 LITERAL@[9; 11)
564 INT_NUMBER@[9; 11) "10"
565 PLUS@[11; 12) "+"
566 LITERAL@[12; 13)
567 INT_NUMBER@[12; 13) "1"
568 SEMI@[13; 14) ";"
569 EXPR_STMT@[14; 15)
570 PATH_EXPR@[14; 15)
571 PATH@[14; 15)
572 PATH_SEGMENT@[14; 15)
573 NAME_REF@[14; 15)
574 IDENT@[14; 15) "a""#,
575 );
576}
577
578// The following tests are port from intellij-rust directly
579// https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt
580
581#[test]
582fn test_path() {
583 let rules = create_rules(
584 r#"
585 macro_rules! foo {
586 ($ i:path) => {
587 fn foo() { let a = $ i; }
588 }
589 }
590"#,
591 );
592 assert_expansion(MacroKind::Items, &rules, "foo! { foo }", "fn foo () {let a = foo ;}");
593 assert_expansion(
594 MacroKind::Items,
595 &rules,
596 "foo! { bar::<u8>::baz::<u8> }",
597 "fn foo () {let a = bar ::< u8 >:: baz ::< u8 > ;}",
598 );
599}
600
601#[test]
602fn test_two_paths() {
603 let rules = create_rules(
604 r#"
605 macro_rules! foo {
606 ($ i:path, $ j:path) => {
607 fn foo() { let a = $ i; let b = $j; }
608 }
609 }
610"#,
611 );
612 assert_expansion(
613 MacroKind::Items,
614 &rules,
615 "foo! { foo, bar }",
616 "fn foo () {let a = foo ; let b = bar ;}",
617 );
618}
619
620#[test]
621fn test_path_with_path() {
622 let rules = create_rules(
623 r#"
624 macro_rules! foo {
625 ($ i:path) => {
626 fn foo() { let a = $ i :: bar; }
627 }
628 }
629"#,
630 );
631 assert_expansion(MacroKind::Items, &rules, "foo! { foo }", "fn foo () {let a = foo :: bar ;}");
632}
633
634#[test]
635fn test_expr() {
636 let rules = create_rules(
637 r#"
638 macro_rules! foo {
639 ($ i:expr) => {
640 fn bar() { $ i; }
641 }
642 }
643"#,
644 );
645
646 assert_expansion(
647 MacroKind::Items,
648 &rules,
649 "foo! { 2 + 2 * baz(3).quux() }",
650 "fn bar () {2 + 2 * baz (3) . quux () ;}",
651 );
652}
653
654#[test]
655fn test_expr_order() {
656 let rules = create_rules(
657 r#"
658 macro_rules! foo {
659 ($ i:expr) => {
660 fn bar() { $ i * 2; }
661 }
662 }
663"#,
664 );
665
666 assert_eq!(
667 expand_to_items(&rules, "foo! { 1 + 1 }").syntax().debug_dump().trim(),
668 r#"MACRO_ITEMS@[0; 15)
669 FN_DEF@[0; 15)
670 FN_KW@[0; 2) "fn"
671 NAME@[2; 5)
672 IDENT@[2; 5) "bar"
673 PARAM_LIST@[5; 7)
674 L_PAREN@[5; 6) "("
675 R_PAREN@[6; 7) ")"
676 BLOCK@[7; 15)
677 L_CURLY@[7; 8) "{"
678 EXPR_STMT@[8; 14)
679 BIN_EXPR@[8; 13)
680 BIN_EXPR@[8; 11)
681 LITERAL@[8; 9)
682 INT_NUMBER@[8; 9) "1"
683 PLUS@[9; 10) "+"
684 LITERAL@[10; 11)
685 INT_NUMBER@[10; 11) "1"
686 STAR@[11; 12) "*"
687 LITERAL@[12; 13)
688 INT_NUMBER@[12; 13) "2"
689 SEMI@[13; 14) ";"
690 R_CURLY@[14; 15) "}""#,
691 );
692}
693
694#[test]
695fn test_last_expr() {
696 let rules = create_rules(
697 r#"
698 macro_rules! vec {
699 ($($item:expr),*) => {
700 {
701 let mut v = Vec::new();
702 $(
703 v.push($item);
704 )*
705 v
706 }
707 };
708 }
709"#,
710 );
711 assert_expansion(
712 MacroKind::Items,
713 &rules,
714 "vec!(1,2,3)",
715 "{let mut v = Vec :: new () ; v . push (1) ; v . push (2) ; v . push (3) ; v}",
716 );
717}
718
719#[test]
720fn test_ty() {
721 let rules = create_rules(
722 r#"
723 macro_rules! foo {
724 ($ i:ty) => (
725 fn bar() -> $ i { unimplemented!() }
726 )
727 }
728"#,
729 );
730 assert_expansion(
731 MacroKind::Items,
732 &rules,
733 "foo! { Baz<u8> }",
734 "fn bar () -> Baz < u8 > {unimplemented ! ()}",
735 );
736}
737
738#[test]
739fn test_ty_with_complex_type() {
740 let rules = create_rules(
741 r#"
742 macro_rules! foo {
743 ($ i:ty) => (
744 fn bar() -> $ i { unimplemented!() }
745 )
746 }
747"#,
748 );
749
750 // Reference lifetime struct with generic type
751 assert_expansion(
752 MacroKind::Items,
753 &rules,
754 "foo! { &'a Baz<u8> }",
755 "fn bar () -> & 'a Baz < u8 > {unimplemented ! ()}",
756 );
757
758 // extern "Rust" func type
759 assert_expansion(
760 MacroKind::Items,
761 &rules,
762 r#"foo! { extern "Rust" fn() -> Ret }"#,
763 r#"fn bar () -> extern "Rust" fn () -> Ret {unimplemented ! ()}"#,
764 );
765}
766
767#[test]
768fn test_pat_() {
769 let rules = create_rules(
770 r#"
771 macro_rules! foo {
772 ($ i:pat) => { fn foo() { let $ i; } }
773 }
774"#,
775 );
776 assert_expansion(MacroKind::Items, &rules, "foo! { (a, b) }", "fn foo () {let (a , b) ;}");
777}
778
779#[test]
780fn test_stmt() {
781 let rules = create_rules(
782 r#"
783 macro_rules! foo {
784 ($ i:stmt) => (
785 fn bar() { $ i; }
786 )
787 }
788"#,
789 );
790 assert_expansion(MacroKind::Items, &rules, "foo! { 2 }", "fn bar () {2 ;}");
791 assert_expansion(MacroKind::Items, &rules, "foo! { let a = 0 }", "fn bar () {let a = 0 ;}");
792}
793
794#[test]
795fn test_single_item() {
796 let rules = create_rules(
797 r#"
798 macro_rules! foo {
799 ($ i:item) => (
800 $ i
801 )
802 }
803"#,
804 );
805 assert_expansion(MacroKind::Items, &rules, "foo! {mod c {}}", "mod c {}");
806}
807
808#[test]
809fn test_all_items() {
810 let rules = create_rules(
811 r#"
812 macro_rules! foo {
813 ($ ($ i:item)*) => ($ (
814 $ i
815 )*)
816 }
817"#,
818 );
819 assert_expansion(MacroKind::Items, &rules, r#"
820 foo! {
821 extern crate a;
822 mod b;
823 mod c {}
824 use d;
825 const E: i32 = 0;
826 static F: i32 = 0;
827 impl G {}
828 struct H;
829 enum I { Foo }
830 trait J {}
831 fn h() {}
832 extern {}
833 type T = u8;
834 }
835"#, r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#);
836}
837
838#[test]
839fn test_block() {
840 let rules = create_rules(
841 r#"
842 macro_rules! foo {
843 ($ i:block) => { fn foo() $ i }
844 }
845"#,
846 );
847 assert_expansion(MacroKind::Stmts, &rules, "foo! { { 1; } }", "fn foo () {1 ;}");
848}
849
850#[test]
851fn test_meta() {
852 let rules = create_rules(
853 r#"
854 macro_rules! foo {
855 ($ i:meta) => (
856 #[$ i]
857 fn bar() {}
858 )
859 }
860"#,
861 );
862 assert_expansion(
863 MacroKind::Items,
864 &rules,
865 r#"foo! { cfg(target_os = "windows") }"#,
866 r#"# [cfg (target_os = "windows")] fn bar () {}"#,
867 );
868}
869
870#[test]
871fn test_meta_doc_comments() {
872 let rules = create_rules(
873 r#"
874 macro_rules! foo {
875 ($(#[$ i:meta])+) => (
876 $(#[$ i])+
877 fn bar() {}
878 )
879 }
880"#,
881 );
882 assert_expansion(
883 MacroKind::Items,
884 &rules,
885 r#"foo! {
886 /// Single Line Doc 1
887 /**
888 MultiLines Doc
889 */
890 }"#,
891 "# [doc = \" Single Line Doc 1\"] # [doc = \" \\\\n MultiLines Doc\\\\n \"] fn bar () {}",
892 );
893}
894
895#[test]
896fn test_tt_block() {
897 let rules = create_rules(
898 r#"
899 macro_rules! foo {
900 ($ i:tt) => { fn foo() $ i }
901 }
902 "#,
903 );
904 assert_expansion(MacroKind::Items, &rules, r#"foo! { { 1; } }"#, r#"fn foo () {1 ;}"#);
905}
906
907#[test]
908fn test_tt_group() {
909 let rules = create_rules(
910 r#"
911 macro_rules! foo {
912 ($($ i:tt)*) => { $($ i)* }
913 }
914 "#,
915 );
916 assert_expansion(MacroKind::Items, &rules, r#"foo! { fn foo() {} }"#, r#"fn foo () {}"#);
917}
918#[test]
919fn test_lifetime() {
920 let rules = create_rules(
921 r#"
922 macro_rules! foo {
923 ($ lt:lifetime) => { struct Ref<$ lt>{ s: &$ lt str } }
924 }
925"#,
926 );
927 assert_expansion(MacroKind::Items, &rules, r#"foo!{'a}"#, r#"struct Ref <'a > {s : &'a str}"#);
928}
929
930#[test]
931fn test_literal() {
932 let rules = create_rules(
933 r#"
934 macro_rules! foo {
935 ($ type:ty $ lit:literal) => { const VALUE: $ type = $ lit;};
936 }
937"#,
938 );
939 assert_expansion(MacroKind::Items, &rules, r#"foo!(u8 0)"#, r#"const VALUE : u8 = 0 ;"#);
940}
941
942#[test]
943fn test_vis() {
944 let rules = create_rules(
945 r#"
946 macro_rules! foo {
947 ($ vis:vis $ name:ident) => { $ vis fn $ name() {}};
948 }
949"#,
950 );
951 assert_expansion(MacroKind::Items, &rules, r#"foo!(pub foo);"#, r#"pub fn foo () {}"#);
952
953 // test optional casse
954 assert_expansion(MacroKind::Items, &rules, r#"foo!(foo);"#, r#"fn foo () {}"#);
955}
956
957#[test]
958fn test_inner_macro_rules() {
959 let rules = create_rules(
960 r#"
961macro_rules! foo {
962 ($a:ident, $b:ident, $c:tt) => {
963
964 macro_rules! bar {
965 ($bi:ident) => {
966 fn $bi() -> u8 {$c}
967 }
968 }
969
970 bar!($a);
971 fn $b() -> u8 {$c}
972 }
973}
974"#,
975 );
976 assert_expansion(
977 MacroKind::Items,
978 &rules,
979 r#"foo!(x,y, 1);"#,
980 r#"macro_rules ! bar {($ bi : ident) => {fn $ bi () -> u8 {1}}} bar ! (x) ; fn y () -> u8 {1}"#,
981 );
982}
983
984// The following tests are based on real world situations
985#[test]
986fn test_vec() {
987 let rules = create_rules(
988 r#"
989 macro_rules! vec {
990 ($($item:expr),*) => {
991 {
992 let mut v = Vec::new();
993 $(
994 v.push($item);
995 )*
996 v
997 }
998 };
999}
1000"#,
1001 );
1002 assert_expansion(MacroKind::Items, &rules, r#"vec!();"#, r#"{let mut v = Vec :: new () ; v}"#);
1003 assert_expansion(
1004 MacroKind::Items,
1005 &rules,
1006 r#"vec![1u32,2]"#,
1007 r#"{let mut v = Vec :: new () ; v . push (1u32) ; v . push (2) ; v}"#,
1008 );
1009
1010 assert_eq!(
1011 expand_to_expr(&rules, r#"vec![1u32,2]"#).syntax().debug_dump().trim(),
1012 r#"BLOCK_EXPR@[0; 45)
1013 BLOCK@[0; 45)
1014 L_CURLY@[0; 1) "{"
1015 LET_STMT@[1; 20)
1016 LET_KW@[1; 4) "let"
1017 BIND_PAT@[4; 8)
1018 MUT_KW@[4; 7) "mut"
1019 NAME@[7; 8)
1020 IDENT@[7; 8) "v"
1021 EQ@[8; 9) "="
1022 CALL_EXPR@[9; 19)
1023 PATH_EXPR@[9; 17)
1024 PATH@[9; 17)
1025 PATH@[9; 12)
1026 PATH_SEGMENT@[9; 12)
1027 NAME_REF@[9; 12)
1028 IDENT@[9; 12) "Vec"
1029 COLONCOLON@[12; 14) "::"
1030 PATH_SEGMENT@[14; 17)
1031 NAME_REF@[14; 17)
1032 IDENT@[14; 17) "new"
1033 ARG_LIST@[17; 19)
1034 L_PAREN@[17; 18) "("
1035 R_PAREN@[18; 19) ")"
1036 SEMI@[19; 20) ";"
1037 EXPR_STMT@[20; 33)
1038 METHOD_CALL_EXPR@[20; 32)
1039 PATH_EXPR@[20; 21)
1040 PATH@[20; 21)
1041 PATH_SEGMENT@[20; 21)
1042 NAME_REF@[20; 21)
1043 IDENT@[20; 21) "v"
1044 DOT@[21; 22) "."
1045 NAME_REF@[22; 26)
1046 IDENT@[22; 26) "push"
1047 ARG_LIST@[26; 32)
1048 L_PAREN@[26; 27) "("
1049 LITERAL@[27; 31)
1050 INT_NUMBER@[27; 31) "1u32"
1051 R_PAREN@[31; 32) ")"
1052 SEMI@[32; 33) ";"
1053 EXPR_STMT@[33; 43)
1054 METHOD_CALL_EXPR@[33; 42)
1055 PATH_EXPR@[33; 34)
1056 PATH@[33; 34)
1057 PATH_SEGMENT@[33; 34)
1058 NAME_REF@[33; 34)
1059 IDENT@[33; 34) "v"
1060 DOT@[34; 35) "."
1061 NAME_REF@[35; 39)
1062 IDENT@[35; 39) "push"
1063 ARG_LIST@[39; 42)
1064 L_PAREN@[39; 40) "("
1065 LITERAL@[40; 41)
1066 INT_NUMBER@[40; 41) "2"
1067 R_PAREN@[41; 42) ")"
1068 SEMI@[42; 43) ";"
1069 PATH_EXPR@[43; 44)
1070 PATH@[43; 44)
1071 PATH_SEGMENT@[43; 44)
1072 NAME_REF@[43; 44)
1073 IDENT@[43; 44) "v"
1074 R_CURLY@[44; 45) "}""#
1075 );
1076}
1077
1078#[test]
1079fn test_winapi_struct() {
1080 // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/macros.rs#L366
1081
1082 let rules = create_rules(
1083 r#"
1084macro_rules! STRUCT {
1085 ($(#[$attrs:meta])* struct $name:ident {
1086 $($field:ident: $ftype:ty,)+
1087 }) => (
1088 #[repr(C)] #[derive(Copy)] $(#[$attrs])*
1089 pub struct $name {
1090 $(pub $field: $ftype,)+
1091 }
1092 impl Clone for $name {
1093 #[inline]
1094 fn clone(&self) -> $name { *self }
1095 }
1096 #[cfg(feature = "impl-default")]
1097 impl Default for $name {
1098 #[inline]
1099 fn default() -> $name { unsafe { $crate::_core::mem::zeroed() } }
1100 }
1101 );
1102}
1103"#,
1104 );
1105 // from https://github.com/retep998/winapi-rs/blob/a7ef2bca086aae76cf6c4ce4c2552988ed9798ad/src/shared/d3d9caps.rs
1106 assert_expansion(MacroKind::Items, &rules, r#"STRUCT!{struct D3DVSHADERCAPS2_0 {Caps: u8,}}"#,
1107 "# [repr (C)] # [derive (Copy)] pub struct D3DVSHADERCAPS2_0 {pub Caps : u8 ,} impl Clone for D3DVSHADERCAPS2_0 {# [inline] fn clone (& self) -> D3DVSHADERCAPS2_0 {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DVSHADERCAPS2_0 {# [inline] fn default () -> D3DVSHADERCAPS2_0 {unsafe {$crate :: _core :: mem :: zeroed ()}}}");
1108 assert_expansion(MacroKind::Items, &rules, r#"STRUCT!{#[cfg_attr(target_arch = "x86", repr(packed))] struct D3DCONTENTPROTECTIONCAPS {Caps : u8 ,}}"#,
1109 "# [repr (C)] # [derive (Copy)] # [cfg_attr (target_arch = \"x86\" , repr (packed))] pub struct D3DCONTENTPROTECTIONCAPS {pub Caps : u8 ,} impl Clone for D3DCONTENTPROTECTIONCAPS {# [inline] fn clone (& self) -> D3DCONTENTPROTECTIONCAPS {* self}} # [cfg (feature = \"impl-default\")] impl Default for D3DCONTENTPROTECTIONCAPS {# [inline] fn default () -> D3DCONTENTPROTECTIONCAPS {unsafe {$crate :: _core :: mem :: zeroed ()}}}");
1110}
1111
1112#[test]
1113fn test_int_base() {
1114 let rules = create_rules(
1115 r#"
1116macro_rules! int_base {
1117 ($Trait:ident for $T:ident as $U:ident -> $Radix:ident) => {
1118 #[stable(feature = "rust1", since = "1.0.0")]
1119 impl fmt::$Trait for $T {
1120 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1121 $Radix.fmt_int(*self as $U, f)
1122 }
1123 }
1124 }
1125}
1126"#,
1127 );
1128
1129 assert_expansion(MacroKind::Items, &rules, r#" int_base!{Binary for isize as usize -> Binary}"#,
1130 "# [stable (feature = \"rust1\" , since = \"1.0.0\")] impl fmt ::Binary for isize {fn fmt (& self , f : & mut fmt :: Formatter < \'_ >) -> fmt :: Result {Binary . fmt_int (* self as usize , f)}}"
1131 );
1132}
1133
1134#[test]
1135fn test_generate_pattern_iterators() {
1136 // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/str/mod.rs
1137 let rules = create_rules(
1138 r#"
1139macro_rules! generate_pattern_iterators {
1140 { double ended; with $(#[$common_stability_attribute:meta])*,
1141 $forward_iterator:ident,
1142 $reverse_iterator:ident, $iterty:ty
1143 } => {
1144 fn foo(){}
1145 }
1146}
1147"#,
1148 );
1149
1150 assert_expansion(MacroKind::Items, &rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str )"#,
1151 "fn foo () {}");
1152}
1153
1154#[test]
1155fn test_impl_fn_for_zst() {
1156 // from https://github.com/rust-lang/rust/blob/5d20ff4d2718c820632b38c1e49d4de648a9810b/src/libcore/internal_macros.rs
1157 let rules = create_rules(
1158 r#"
1159macro_rules! impl_fn_for_zst {
1160 { $( $( #[$attr: meta] )*
1161 struct $Name: ident impl$( <$( $lifetime : lifetime ),+> )? Fn =
1162 |$( $arg: ident: $ArgTy: ty ),*| -> $ReturnTy: ty
1163$body: block; )+
1164 } => {
1165 $(
1166 $( #[$attr] )*
1167 struct $Name;
1168
1169 impl $( <$( $lifetime ),+> )? Fn<($( $ArgTy, )*)> for $Name {
1170 #[inline]
1171 extern "rust-call" fn call(&self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
1172 $body
1173 }
1174 }
1175
1176 impl $( <$( $lifetime ),+> )? FnMut<($( $ArgTy, )*)> for $Name {
1177 #[inline]
1178 extern "rust-call" fn call_mut(
1179 &mut self,
1180 ($( $arg, )*): ($( $ArgTy, )*)
1181 ) -> $ReturnTy {
1182 Fn::call(&*self, ($( $arg, )*))
1183 }
1184 }
1185
1186 impl $( <$( $lifetime ),+> )? FnOnce<($( $ArgTy, )*)> for $Name {
1187 type Output = $ReturnTy;
1188
1189 #[inline]
1190 extern "rust-call" fn call_once(self, ($( $arg, )*): ($( $ArgTy, )*)) -> $ReturnTy {
1191 Fn::call(&self, ($( $arg, )*))
1192 }
1193 }
1194 )+
1195}
1196 }
1197}
1198"#,
1199 );
1200
1201 assert_expansion(MacroKind::Items, &rules, r#"
1202impl_fn_for_zst ! {
1203 # [ derive ( Clone ) ]
1204 struct CharEscapeDebugContinue impl Fn = | c : char | -> char :: EscapeDebug {
1205 c . escape_debug_ext ( false )
1206 } ;
1207
1208 # [ derive ( Clone ) ]
1209 struct CharEscapeUnicode impl Fn = | c : char | -> char :: EscapeUnicode {
1210 c . escape_unicode ( )
1211 } ;
1212 # [ derive ( Clone ) ]
1213 struct CharEscapeDefault impl Fn = | c : char | -> char :: EscapeDefault {
1214 c . escape_default ( )
1215 } ;
1216 }
1217"#,
1218 "# [derive (Clone)] struct CharEscapeDebugContinue ; impl Fn < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDebug {{c . escape_debug_ext (false)}}} impl FnMut < (char ,) > for CharEscapeDebugContinue {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDebugContinue {type Output = char :: EscapeDebug ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDebug {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeUnicode ; impl Fn < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeUnicode {{c . escape_unicode ()}}} impl FnMut < (char ,) > for CharEscapeUnicode {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeUnicode {type Output = char :: EscapeUnicode ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeUnicode {Fn :: call (& self , (c ,))}} # [derive (Clone)] struct CharEscapeDefault ; impl Fn < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call (& self , (c ,) : (char ,)) -> char :: EscapeDefault {{c . escape_default ()}}} impl FnMut < (char ,) > for CharEscapeDefault {# [inline] extern \"rust-call\" fn call_mut (& mut self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (&* self , (c ,))}} impl FnOnce < (char ,) > for CharEscapeDefault {type Output = char :: EscapeDefault ; # [inline] extern \"rust-call\" fn call_once (self , (c ,) : (char ,)) -> char :: EscapeDefault {Fn :: call (& self , (c ,))}}");
1219}
1220
1221#[test]
1222fn test_impl_nonzero_fmt() {
1223 // from https://github.com/rust-lang/rust/blob/316a391dcb7d66dc25f1f9a4ec9d368ef7615005/src/libcore/num/mod.rs#L12
1224 let rules = create_rules(
1225 r#"
1226 macro_rules! impl_nonzero_fmt {
1227 ( #[$stability: meta] ( $( $Trait: ident ),+ ) for $Ty: ident ) => {
1228 fn foo () {}
1229 }
1230 }
1231"#,
1232 );
1233
1234 assert_expansion(MacroKind::Items, &rules, r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#,
1235 "fn foo () {}");
1236}
1237
1238#[test]
1239fn test_cfg_if_items() {
1240 // from https://github.com/rust-lang/rust/blob/33fe1131cadba69d317156847be9a402b89f11bb/src/libstd/macros.rs#L986
1241 let rules = create_rules(
1242 r#"
1243 macro_rules! __cfg_if_items {
1244 (($($not:meta,)*) ; ) => {};
1245 (($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => {
1246 __cfg_if_items! { ($($not,)* $($m,)*) ; $($rest)* }
1247 }
1248 }
1249"#,
1250 );
1251
1252 assert_expansion(MacroKind::Items, &rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#,
1253 "__cfg_if_items ! {(rustdoc ,) ;}");
1254}
1255
1256#[test]
1257fn test_cfg_if_main() {
1258 // from https://github.com/rust-lang/rust/blob/3d211248393686e0f73851fc7548f6605220fbe1/src/libpanic_unwind/macros.rs#L9
1259 let rules = create_rules(
1260 r#"
1261 macro_rules! cfg_if {
1262 ($(
1263 if #[cfg($($meta:meta),*)] { $($it:item)* }
1264 ) else * else {
1265 $($it2:item)*
1266 }) => {
1267 __cfg_if_items! {
1268 () ;
1269 $( ( ($($meta),*) ($($it)*) ), )*
1270 ( () ($($it2)*) ),
1271 }
1272 };
1273
1274 // Internal macro to Apply a cfg attribute to a list of items
1275 (@__apply $m:meta, $($it:item)*) => {
1276 $(#[$m] $it)*
1277 };
1278 }
1279"#,
1280 );
1281
1282 assert_expansion(MacroKind::Items, &rules, r#"
1283cfg_if ! {
1284 if # [ cfg ( target_env = "msvc" ) ] {
1285 // no extra unwinder support needed
1286 } else if # [ cfg ( all ( target_arch = "wasm32" , not ( target_os = "emscripten" ) ) ) ] {
1287 // no unwinder on the system!
1288 } else {
1289 mod libunwind ;
1290 pub use libunwind :: * ;
1291 }
1292 }
1293"#,
1294 "__cfg_if_items ! {() ; ((target_env = \"msvc\") ()) , ((all (target_arch = \"wasm32\" , not (target_os = \"emscripten\"))) ()) , (() (mod libunwind ; pub use libunwind :: * ;)) ,}");
1295
1296 assert_expansion(MacroKind::Items, &rules, r#"
1297cfg_if ! { @ __apply cfg ( all ( not ( any ( not ( any ( target_os = "solaris" , target_os = "illumos" ) ) ) ) ) ) , }
1298"#,
1299 ""
1300 );
1301}
1302
1303#[test]
1304fn test_proptest_arbitrary() {
1305 // from https://github.com/AltSysrq/proptest/blob/d1c4b049337d2f75dd6f49a095115f7c532e5129/proptest/src/arbitrary/macros.rs#L16
1306 let rules = create_rules(
1307 r#"
1308macro_rules! arbitrary {
1309 ([$($bounds : tt)*] $typ: ty, $strat: ty, $params: ty;
1310 $args: ident => $logic: expr) => {
1311 impl<$($bounds)*> $crate::arbitrary::Arbitrary for $typ {
1312 type Parameters = $params;
1313 type Strategy = $strat;
1314 fn arbitrary_with($args: Self::Parameters) -> Self::Strategy {
1315 $logic
1316 }
1317 }
1318 };
1319
1320}"#,
1321 );
1322
1323 assert_expansion(MacroKind::Items, &rules, r#"arbitrary ! ( [ A : Arbitrary ]
1324 Vec < A > ,
1325 VecStrategy < A :: Strategy > ,
1326 RangedParams1 < A :: Parameters > ;
1327 args => { let product_unpack ! [ range , a ] = args ; vec ( any_with :: < A > ( a ) , range ) }
1328 ) ;"#,
1329 "impl <A : Arbitrary > $crate :: arbitrary :: Arbitrary for Vec < A > {type Parameters = RangedParams1 < A :: Parameters > ; type Strategy = VecStrategy < A :: Strategy > ; fn arbitrary_with (args : Self :: Parameters) -> Self :: Strategy {{let product_unpack ! [range , a] = args ; vec (any_with :: < A > (a) , range)}}}");
1330}
1331
1332#[test]
1333fn test_old_ridl() {
1334 // This is from winapi 2.8, which do not have a link from github
1335 //
1336 let rules = create_rules(
1337 r#"
1338#[macro_export]
1339macro_rules! RIDL {
1340 (interface $interface:ident ($vtbl:ident) : $pinterface:ident ($pvtbl:ident)
1341 {$(
1342 fn $method:ident(&mut self $(,$p:ident : $t:ty)*) -> $rtr:ty
1343 ),+}
1344 ) => {
1345 impl $interface {
1346 $(pub unsafe fn $method(&mut self) -> $rtr {
1347 ((*self.lpVtbl).$method)(self $(,$p)*)
1348 })+
1349 }
1350 };
1351}"#,
1352 );
1353
1354 let expanded = expand(&rules, r#"
1355RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID3D11DeviceChildVtbl) {
1356 fn GetDataSize(&mut self) -> UINT
1357}}"#);
1358 assert_eq!(expanded.to_string(), "impl ID3D11Asynchronous {pub unsafe fn GetDataSize (& mut self) -> UINT {((* self . lpVtbl) .GetDataSize) (self)}}");
1359}
1360
1361#[test]
1362fn test_quick_error() {
1363 let rules = create_rules(
1364 r#"
1365macro_rules! quick_error {
1366
1367 (SORT [enum $name:ident $( #[$meta:meta] )*]
1368 items [$($( #[$imeta:meta] )*
1369 => $iitem:ident: $imode:tt [$( $ivar:ident: $ityp:ty ),*]
1370 {$( $ifuncs:tt )*} )* ]
1371 buf [ ]
1372 queue [ ]
1373 ) => {
1374 quick_error!(ENUM_DEFINITION [enum $name $( #[$meta] )*]
1375 body []
1376 queue [$(
1377 $( #[$imeta] )*
1378 =>
1379 $iitem: $imode [$( $ivar: $ityp ),*]
1380 )*]
1381 );
1382};
1383
1384}
1385"#,
1386 );
1387
1388 let expanded = expand(
1389 &rules,
1390 r#"
1391quick_error ! (SORT [enum Wrapped # [derive (Debug)]] items [
1392 => One : UNIT [] {}
1393 => Two : TUPLE [s :String] {display ("two: {}" , s) from ()}
1394 ] buf [] queue []) ;
1395"#,
1396 );
1397
1398 assert_eq!(expanded.to_string(), "quick_error ! (ENUM_DEFINITION [enum Wrapped # [derive (Debug)]] body [] queue [=> One : UNIT [] => Two : TUPLE [s : String]]) ;");
1399}
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs
index eef642a9c..d85ab43e4 100644
--- a/crates/ra_mbe/src/tt_cursor.rs
+++ b/crates/ra_mbe/src/tt_cursor.rs
@@ -1,6 +1,5 @@
1use crate::ParseError; 1use crate::ParseError;
2use crate::subtree_parser::Parser; 2use crate::subtree_parser::Parser;
3use crate::subtree_source::TokenPeek;
4use smallvec::{SmallVec, smallvec}; 3use smallvec::{SmallVec, smallvec};
5 4
6#[derive(Debug, Clone)] 5#[derive(Debug, Clone)]
@@ -150,9 +149,16 @@ impl<'a> TtCursor<'a> {
150 self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into()) 149 self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into())
151 } 150 }
152 151
153 pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> { 152 pub(crate) fn try_eat_vis(&mut self) -> Option<tt::TokenTree> {
153 // `vis` matcher is optional
154 let old_pos = self.pos;
154 let parser = Parser::new(&mut self.pos, self.subtree); 155 let parser = Parser::new(&mut self.pos, self.subtree);
155 parser.parse_vis() 156
157 let res = parser.parse_vis();
158 if res.is_none() {
159 self.pos = old_pos;
160 }
161 res
156 } 162 }
157 163
158 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { 164 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> {
@@ -262,3 +268,48 @@ impl<'a> TtCursor<'a> {
262 self.pos = memento.pos; 268 self.pos = memento.pos;
263 } 269 }
264} 270}
271
272pub(crate) struct TokenPeek<'a, I>
273where
274 I: Iterator<Item = &'a tt::TokenTree>,
275{
276 iter: itertools::MultiPeek<I>,
277}
278
279// helper function
280fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
281 if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
282 return Some(pp);
283 }
284 None
285}
286
287impl<'a, I> TokenPeek<'a, I>
288where
289 I: Iterator<Item = &'a tt::TokenTree>,
290{
291 pub fn new(iter: I) -> Self {
292 TokenPeek { iter: itertools::multipeek(iter) }
293 }
294
295 pub fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> {
296 if p.spacing != tt::Spacing::Joint {
297 return None;
298 }
299
300 self.iter.reset_peek();
301 let p1 = to_punct(self.iter.peek()?)?;
302 Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
303 }
304
305 pub fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> {
306 self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
307 if !last_joint {
308 None
309 } else {
310 let p2 = to_punct(*self.iter.peek()?)?;
311 Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
312 }
313 })
314 }
315}
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-09 14:21:21 +0100