From 3ff5440a503f090032136c37c3d44375d6107db1 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 19 Apr 2019 02:47:29 +0800 Subject: Add MacroItems and MacroStmts in grammer.ron --- crates/ra_mbe/src/lib.rs | 22 ++++++++++++++-------- crates/ra_mbe/src/syntax_bridge.rs | 23 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 2f47e32d3..4cfa1f955 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -37,9 +37,15 @@ pub enum ExpandError { NoMatchingRule, UnexpectedToken, BindingError(String), + ConversionError, } -pub use crate::syntax_bridge::{ast_to_token_tree, token_tree_to_ast_item_list, syntax_node_to_token_tree}; +pub use crate::syntax_bridge::{ + ast_to_token_tree, + token_tree_to_ast_item_list, + syntax_node_to_token_tree, + token_tree_to_macro_items, +}; /// This struct contains AST for a single `macro_rules` definition. What might /// be very confusing is that AST has almost exactly the same shape as @@ -192,21 +198,21 @@ impl_froms!(TokenTree: Leaf, Subtree); pub(crate) fn expand_to_syntax( rules: &MacroRules, invocation: &str, - ) -> ra_syntax::TreeArc { + ) -> ra_syntax::TreeArc { let expanded = expand(rules, invocation); - token_tree_to_ast_item_list(&expanded) + token_tree_to_macro_items(&expanded) } pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { let expanded = expand(rules, invocation); assert_eq!(expanded.to_string(), expansion); - let tree = token_tree_to_ast_item_list(&expanded); + let tree = token_tree_to_macro_items(&expanded); // Eat all white space by parse it back and forth let expansion = ast::SourceFile::parse(expansion); let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0; - let file = token_tree_to_ast_item_list(&expansion); + let file = token_tree_to_macro_items(&expansion); assert_eq!(tree.syntax().debug_dump().trim(), file.syntax().debug_dump().trim()); } @@ -346,11 +352,11 @@ impl_froms!(TokenTree: Leaf, Subtree); ", ); let expansion = expand(&rules, "structs!(Foo, Bar)"); - let tree = token_tree_to_ast_item_list(&expansion); + let tree = token_tree_to_macro_items(&expansion); assert_eq!( tree.syntax().debug_dump().trim(), r#" -SOURCE_FILE@[0; 40) +MACRO_ITEMS@[0; 40) STRUCT_DEF@[0; 20) STRUCT_KW@[0; 6) "struct" NAME@[6; 9) @@ -527,7 +533,7 @@ SOURCE_FILE@[0; 40) assert_eq!( expand_to_syntax(&rules, "foo! { 1 + 1 }").syntax().debug_dump().trim(), - r#"SOURCE_FILE@[0; 15) + r#"MACRO_ITEMS@[0; 15) FN_DEF@[0; 15) FN_KW@[0; 2) "fn" NAME@[2; 5) diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 28ded7870..19e09be80 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -30,6 +30,29 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke Some((tt, token_map)) } +// The following items are what `rustc` macro can be parsed into : +// link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141 +// * Expr(P) +// * Pat(P) +// * Ty(P) +// * Stmts(SmallVec<[ast::Stmt; 1]>) +// * Items(SmallVec<[P; 1]>) +// +// * TraitItems(SmallVec<[ast::TraitItem; 1]>) +// * ImplItems(SmallVec<[ast::ImplItem; 1]>) +// * ForeignItems(SmallVec<[ast::ForeignItem; 1]> +// +// + +/// Parses the token tree (result of macro expansion) as a sequence of items +pub fn token_tree_to_macro_items(tt: &tt::Subtree) -> TreeArc { + let token_source = SubtreeTokenSource::new(tt); + let mut tree_sink = TtTreeSink::new(token_source.querier()); + ra_parser::parse_macro_items(&token_source, &mut tree_sink); + let syntax = tree_sink.inner.finish(); + ast::MacroItems::cast(&syntax).unwrap().to_owned() +} + /// Parses the token tree (result of macro expansion) as a sequence of items pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc { let token_source = SubtreeTokenSource::new(tt); -- cgit v1.2.3 From c0f19d70056fada5f381019694d893e0ffe8360a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 19 Apr 2019 03:49:56 +0800 Subject: Add expr, pat, ty and macro_stmts --- crates/ra_mbe/src/lib.rs | 57 ++++++++++++++++++++++++++++++++++++++ crates/ra_mbe/src/syntax_bridge.rs | 46 ++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 5 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 4cfa1f955..a29b07aba 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -44,7 +44,11 @@ pub use crate::syntax_bridge::{ ast_to_token_tree, token_tree_to_ast_item_list, syntax_node_to_token_tree, + token_tree_to_expr, + token_tree_to_pat, + token_tree_to_ty, token_tree_to_macro_items, + token_tree_to_macro_stmts, }; /// This struct contains AST for a single `macro_rules` definition. What might @@ -450,6 +454,59 @@ MACRO_ITEMS@[0; 40) assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); } + #[test] + fn test_tt_to_stmts() { + let rules = create_rules( + r#" + macro_rules! foo { + () => { + let a = 0; + a = 10 + 1; + a + } + } +"#, + ); + + let expanded = expand(&rules, "foo!{}"); + let stmts = token_tree_to_macro_stmts(&expanded); + + assert_eq!( + stmts.syntax().debug_dump().trim(), + r#"MACRO_STMTS@[0; 15) + LET_STMT@[0; 7) + LET_KW@[0; 3) "let" + BIND_PAT@[3; 4) + NAME@[3; 4) + IDENT@[3; 4) "a" + EQ@[4; 5) "=" + LITERAL@[5; 6) + INT_NUMBER@[5; 6) "0" + SEMI@[6; 7) ";" + EXPR_STMT@[7; 14) + BIN_EXPR@[7; 13) + PATH_EXPR@[7; 8) + PATH@[7; 8) + PATH_SEGMENT@[7; 8) + NAME_REF@[7; 8) + IDENT@[7; 8) "a" + EQ@[8; 9) "=" + BIN_EXPR@[9; 13) + LITERAL@[9; 11) + INT_NUMBER@[9; 11) "10" + PLUS@[11; 12) "+" + LITERAL@[12; 13) + INT_NUMBER@[12; 13) "1" + SEMI@[13; 14) ";" + EXPR_STMT@[14; 15) + PATH_EXPR@[14; 15) + PATH@[14; 15) + PATH_SEGMENT@[14; 15) + NAME_REF@[14; 15) + IDENT@[14; 15) "a""#, + ); + } + // The following tests are port from intellij-rust directly // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 19e09be80..6af3b1995 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -32,11 +32,11 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke // The following items are what `rustc` macro can be parsed into : // link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141 -// * Expr(P) -// * Pat(P) -// * Ty(P) -// * Stmts(SmallVec<[ast::Stmt; 1]>) -// * Items(SmallVec<[P; 1]>) +// * Expr(P) -> token_tree_to_expr +// * Pat(P) -> token_tree_to_pat +// * Ty(P) -> token_tree_to_ty +// * Stmts(SmallVec<[ast::Stmt; 1]>) -> token_tree_to_stmts +// * Items(SmallVec<[P; 1]>) -> token_tree_to_items // // * TraitItems(SmallVec<[ast::TraitItem; 1]>) // * ImplItems(SmallVec<[ast::ImplItem; 1]>) @@ -44,6 +44,42 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke // // +/// Parses the token tree (result of macro expansion) to an expression +pub fn token_tree_to_expr(tt: &tt::Subtree) -> TreeArc { + let token_source = SubtreeTokenSource::new(tt); + let mut tree_sink = TtTreeSink::new(token_source.querier()); + ra_parser::parse_expr(&token_source, &mut tree_sink); + let syntax = tree_sink.inner.finish(); + ast::Expr::cast(&syntax).unwrap().to_owned() +} + +/// Parses the token tree (result of macro expansion) to a Pattern +pub fn token_tree_to_pat(tt: &tt::Subtree) -> TreeArc { + let token_source = SubtreeTokenSource::new(tt); + let mut tree_sink = TtTreeSink::new(token_source.querier()); + ra_parser::parse_pat(&token_source, &mut tree_sink); + let syntax = tree_sink.inner.finish(); + ast::Pat::cast(&syntax).unwrap().to_owned() +} + +/// Parses the token tree (result of macro expansion) to a Type +pub fn token_tree_to_ty(tt: &tt::Subtree) -> TreeArc { + let token_source = SubtreeTokenSource::new(tt); + let mut tree_sink = TtTreeSink::new(token_source.querier()); + ra_parser::parse_ty(&token_source, &mut tree_sink); + let syntax = tree_sink.inner.finish(); + ast::TypeRef::cast(&syntax).unwrap().to_owned() +} + +/// Parses the token tree (result of macro expansion) as a sequence of stmts +pub fn token_tree_to_macro_stmts(tt: &tt::Subtree) -> TreeArc { + let token_source = SubtreeTokenSource::new(tt); + let mut tree_sink = TtTreeSink::new(token_source.querier()); + ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); + let syntax = tree_sink.inner.finish(); + ast::MacroStmts::cast(&syntax).unwrap().to_owned() +} + /// Parses the token tree (result of macro expansion) as a sequence of items pub fn token_tree_to_macro_items(tt: &tt::Subtree) -> TreeArc { let token_source = SubtreeTokenSource::new(tt); -- cgit v1.2.3 From a6d51e09610989821aaf79871bcab0661c9b0f74 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Fri, 19 Apr 2019 04:03:22 +0800 Subject: Return Result for token_tree_to_xx functions --- crates/ra_mbe/src/lib.rs | 11 +++++++---- crates/ra_mbe/src/syntax_bridge.rs | 27 +++++++++++++++++---------- 2 files changed, 24 insertions(+), 14 deletions(-) (limited to 'crates/ra_mbe/src') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index a29b07aba..3a9db7835 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -204,7 +204,7 @@ impl_froms!(TokenTree: Leaf, Subtree); invocation: &str, ) -> ra_syntax::TreeArc { let expanded = expand(rules, invocation); - token_tree_to_macro_items(&expanded) + token_tree_to_macro_items(&expanded).unwrap() } pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { @@ -218,7 +218,10 @@ impl_froms!(TokenTree: Leaf, Subtree); let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0; let file = token_tree_to_macro_items(&expansion); - assert_eq!(tree.syntax().debug_dump().trim(), file.syntax().debug_dump().trim()); + assert_eq!( + tree.unwrap().syntax().debug_dump().trim(), + file.unwrap().syntax().debug_dump().trim() + ); } #[test] @@ -358,7 +361,7 @@ impl_froms!(TokenTree: Leaf, Subtree); let expansion = expand(&rules, "structs!(Foo, Bar)"); let tree = token_tree_to_macro_items(&expansion); assert_eq!( - tree.syntax().debug_dump().trim(), + tree.unwrap().syntax().debug_dump().trim(), r#" MACRO_ITEMS@[0; 40) STRUCT_DEF@[0; 20) @@ -472,7 +475,7 @@ MACRO_ITEMS@[0; 40) let stmts = token_tree_to_macro_stmts(&expanded); assert_eq!( - stmts.syntax().debug_dump().trim(), + stmts.unwrap().syntax().debug_dump().trim(), r#"MACRO_STMTS@[0; 15) LET_STMT@[0; 7) LET_KW@[0; 3) "let" diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 6af3b1995..5844d3f12 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs @@ -5,6 +5,7 @@ use ra_syntax::{ }; use crate::subtree_source::{SubtreeTokenSource, Querier}; +use crate::ExpandError; /// Maps `tt::TokenId` to the relative range of the original token. #[derive(Default)] @@ -45,48 +46,54 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke // /// Parses the token tree (result of macro expansion) to an expression -pub fn token_tree_to_expr(tt: &tt::Subtree) -> TreeArc { +pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result, ExpandError> { let token_source = SubtreeTokenSource::new(tt); let mut tree_sink = TtTreeSink::new(token_source.querier()); ra_parser::parse_expr(&token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); - ast::Expr::cast(&syntax).unwrap().to_owned() + ast::Expr::cast(&syntax) + .map(|m| m.to_owned()) + .ok_or_else(|| crate::ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) to a Pattern -pub fn token_tree_to_pat(tt: &tt::Subtree) -> TreeArc { +pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result, ExpandError> { let token_source = SubtreeTokenSource::new(tt); let mut tree_sink = TtTreeSink::new(token_source.querier()); ra_parser::parse_pat(&token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); - ast::Pat::cast(&syntax).unwrap().to_owned() + ast::Pat::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) to a Type -pub fn token_tree_to_ty(tt: &tt::Subtree) -> TreeArc { +pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result, ExpandError> { let token_source = SubtreeTokenSource::new(tt); let mut tree_sink = TtTreeSink::new(token_source.querier()); ra_parser::parse_ty(&token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); - ast::TypeRef::cast(&syntax).unwrap().to_owned() + ast::TypeRef::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) as a sequence of stmts -pub fn token_tree_to_macro_stmts(tt: &tt::Subtree) -> TreeArc { +pub fn token_tree_to_macro_stmts( + tt: &tt::Subtree, +) -> Result, ExpandError> { let token_source = SubtreeTokenSource::new(tt); let mut tree_sink = TtTreeSink::new(token_source.querier()); ra_parser::parse_macro_stmts(&token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); - ast::MacroStmts::cast(&syntax).unwrap().to_owned() + ast::MacroStmts::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) as a sequence of items -pub fn token_tree_to_macro_items(tt: &tt::Subtree) -> TreeArc { +pub fn token_tree_to_macro_items( + tt: &tt::Subtree, +) -> Result, ExpandError> { let token_source = SubtreeTokenSource::new(tt); let mut tree_sink = TtTreeSink::new(token_source.querier()); ra_parser::parse_macro_items(&token_source, &mut tree_sink); let syntax = tree_sink.inner.finish(); - ast::MacroItems::cast(&syntax).unwrap().to_owned() + ast::MacroItems::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError) } /// Parses the token tree (result of macro expansion) as a sequence of items -- cgit v1.2.3