3 files changed, 111 insertions, 73 deletions

diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index 15f000175..8a31d1c36 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -31,8 +31,7 @@ pub enum ExpandError {
 }
 
 pub use crate::syntax_bridge::{
-    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_expr, token_tree_to_items,
-    token_tree_to_macro_stmts, token_tree_to_pat, token_tree_to_ty,
+    ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, RevTokenMap, TokenMap,
 };
 
 /// This struct contains AST for a single `macro_rules` definition. What might
@@ -118,6 +117,10 @@ impl MacroRules {
         shift_subtree(&mut tt, self.shift);
         mbe_expander::expand(self, &tt)
     }
+
+    pub fn shift(&self) -> u32 {
+        self.shift
+    }
 }
 
 impl Rule {
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 592fcf527..3f57ce3b5 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -1,9 +1,6 @@
 //! FIXME: write short doc here
 
-use ra_parser::{
-    FragmentKind::{self, *},
-    ParseError, TreeSink,
-};
+use ra_parser::{FragmentKind, ParseError, TreeSink};
 use ra_syntax::{
     ast, AstNode, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
     SyntaxTreeBuilder, TextRange, TextUnit, T,
@@ -14,12 +11,18 @@ use crate::subtree_source::SubtreeTokenSource;
 use crate::ExpandError;
 
 /// Maps `tt::TokenId` to the relative range of the original token.
-#[derive(Default)]
+#[derive(Debug, PartialEq, Eq, Default)]
 pub struct TokenMap {
     /// Maps `tt::TokenId` to the *relative* source range.
     tokens: Vec<TextRange>,
 }
 
+/// Maps relative range of the expanded syntax node to `tt::TokenId`
+#[derive(Debug, PartialEq, Eq, Default)]
+pub struct RevTokenMap {
+    pub ranges: Vec<(TextRange, tt::TokenId)>,
+}
+
 /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro
 /// will consume).
 pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> {
@@ -49,10 +52,10 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
 // * ImplItems(SmallVec<[ast::ImplItem; 1]>)
 // * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
 
-fn fragment_to_syntax_node(
+pub fn token_tree_to_syntax_node(
     tt: &tt::Subtree,
     fragment_kind: FragmentKind,
-) -> Result<Parse<SyntaxNode>, ExpandError> {
+) -> Result<(Parse<SyntaxNode>, RevTokenMap), ExpandError> {
     let tmp;
     let tokens = match tt {
         tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(),
@@ -69,38 +72,8 @@ fn fragment_to_syntax_node(
         return Err(ExpandError::ConversionError);
     }
     //FIXME: would be cool to report errors
-    let parse = tree_sink.inner.finish();
-    Ok(parse)
-}
-
-/// Parses the token tree (result of macro expansion) to an expression
-pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<Parse<ast::Expr>, ExpandError> {
-    let parse = fragment_to_syntax_node(tt, Expr)?;
-    parse.cast().ok_or_else(|| crate::ExpandError::ConversionError)
-}
-
-/// Parses the token tree (result of macro expansion) to a Pattern
-pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<Parse<ast::Pat>, ExpandError> {
-    let parse = fragment_to_syntax_node(tt, Pattern)?;
-    parse.cast().ok_or_else(|| crate::ExpandError::ConversionError)
-}
-
-/// Parses the token tree (result of macro expansion) to a Type
-pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<Parse<ast::TypeRef>, ExpandError> {
-    let parse = fragment_to_syntax_node(tt, Type)?;
-    parse.cast().ok_or_else(|| crate::ExpandError::ConversionError)
-}
-
-/// Parses the token tree (result of macro expansion) as a sequence of stmts
-pub fn token_tree_to_macro_stmts(tt: &tt::Subtree) -> Result<Parse<ast::MacroStmts>, ExpandError> {
-    let parse = fragment_to_syntax_node(tt, Statements)?;
-    parse.cast().ok_or_else(|| crate::ExpandError::ConversionError)
-}
-
-/// Parses the token tree (result of macro expansion) as a sequence of items
-pub fn token_tree_to_items(tt: &tt::Subtree) -> Result<Parse<ast::MacroItems>, ExpandError> {
-    let parse = fragment_to_syntax_node(tt, Items)?;
-    parse.cast().ok_or_else(|| crate::ExpandError::ConversionError)
+    let (parse, range_map) = tree_sink.finish();
+    Ok((parse, range_map))
 }
 
 impl TokenMap {
@@ -116,6 +89,12 @@ impl TokenMap {
     }
 }
 
+impl RevTokenMap {
+    fn add(&mut self, relative_range: TextRange, token_id: tt::TokenId) {
+        self.ranges.push((relative_range, token_id.clone()))
+    }
+}
+
 /// Returns the textual content of a doc comment block as a quoted string
 /// That is, strips leading `///` (or `/**`, etc)
 /// and strips the ending `*/`
@@ -262,6 +241,7 @@ struct TtTreeSink<'a> {
     cursor: Cursor<'a>,
     text_pos: TextUnit,
     inner: SyntaxTreeBuilder,
+    range_map: RevTokenMap,
 
     // Number of roots
     // Use for detect ill-form tree which is not single root
@@ -276,8 +256,13 @@ impl<'a> TtTreeSink<'a> {
             text_pos: 0.into(),
             inner: SyntaxTreeBuilder::default(),
             roots: smallvec::SmallVec::new(),
+            range_map: RevTokenMap::default(),
         }
     }
+
+    fn finish(self) -> (Parse<SyntaxNode>, RevTokenMap) {
+        (self.inner.finish(), self.range_map)
+    }
 }
 
 fn delim_to_str(d: tt::Delimiter, closing: bool) -> SmolStr {
@@ -307,6 +292,15 @@ impl<'a> TreeSink for TtTreeSink<'a> {
 
             match self.cursor.token_tree() {
                 Some(tt::TokenTree::Leaf(leaf)) => {
+                    // Mark the range if needed
+                    if let tt::Leaf::Ident(ident) = leaf {
+                        if kind == IDENT {
+                            let range =
+                                TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text));
+                            self.range_map.add(range, ident.id);
+                        }
+                    }
+
                     self.cursor = self.cursor.bump();
                     self.buf += &format!("{}", leaf);
                 }
@@ -337,6 +331,7 @@ impl<'a> TreeSink for TtTreeSink<'a> {
         {
             if curr.spacing == tt::Spacing::Alone {
                 self.inner.token(WHITESPACE, " ".into());
+                self.text_pos += TextUnit::of_char(' ');
             }
         }
     }
@@ -423,6 +418,6 @@ mod tests {
             "#,
         );
         let expansion = expand(&rules, "stmts!();");
-        assert!(token_tree_to_expr(&expansion).is_err());
+        assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err());
     }
 }
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index a23e3afe3..0109a4d98 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -1,3 +1,4 @@
+use ra_parser::FragmentKind;
 use ra_syntax::{ast, AstNode, NodeOrToken, WalkEvent};
 use test_utils::assert_eq_text;
 
@@ -126,9 +127,9 @@ fn test_expr_order() {
 "#,
     );
     let expanded = expand(&rules, "foo! { 1 + 1}");
-    let tree = token_tree_to_items(&expanded).unwrap().tree();
+    let tree = token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node();
 
-    let dump = format!("{:#?}", tree.syntax());
+    let dump = format!("{:#?}", tree);
     assert_eq_text!(
         dump.trim(),
         r#"MACRO_ITEMS@[0; 15)
@@ -383,9 +384,9 @@ fn test_expand_to_item_list() {
             ",
     );
     let expansion = expand(&rules, "structs!(Foo, Bar);");
-    let tree = token_tree_to_items(&expansion).unwrap().tree();
+    let tree = token_tree_to_syntax_node(&expansion, FragmentKind::Items).unwrap().0.syntax_node();
     assert_eq!(
-        format!("{:#?}", tree.syntax()).trim(),
+        format!("{:#?}", tree).trim(),
         r#"
 MACRO_ITEMS@[0; 40)
   STRUCT_DEF@[0; 20)
@@ -501,10 +502,11 @@ fn test_tt_to_stmts() {
     );
 
     let expanded = expand(&rules, "foo!{}");
-    let stmts = token_tree_to_macro_stmts(&expanded).unwrap().tree();
+    let stmts =
+        token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node();
 
     assert_eq!(
-        format!("{:#?}", stmts.syntax()).trim(),
+        format!("{:#?}", stmts).trim(),
         r#"MACRO_STMTS@[0; 15)
   LET_STMT@[0; 7)
     LET_KW@[0; 3) "let"
@@ -754,7 +756,10 @@ fn test_all_items() {
         }
 "#,
     );
-    assert_expansion(MacroKind::Items, &rules, r#"
+    assert_expansion(
+        MacroKind::Items,
+        &rules,
+        r#"
         foo! {
             extern crate a;
             mod b;
@@ -770,7 +775,9 @@ fn test_all_items() {
             extern {}
             type T = u8;
         }
-"#, r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#);
+"#,
+        r#"extern crate a ; mod b ; mod c {} use d ; const E : i32 = 0 ; static F : i32 = 0 ; impl G {} struct H ; enum I {Foo} trait J {} fn h () {} extern {} type T = u8 ;"#,
+    );
 }
 
 #[test]
@@ -946,10 +953,10 @@ fn test_vec() {
     );
 
     let expansion = expand(&rules, r#"vec![1u32,2];"#);
-    let tree = token_tree_to_expr(&expansion).unwrap().tree();
+    let tree = token_tree_to_syntax_node(&expansion, FragmentKind::Expr).unwrap().0.syntax_node();
 
     assert_eq!(
-        format!("{:#?}", tree.syntax()).trim(),
+        format!("{:#?}", tree).trim(),
         r#"BLOCK_EXPR@[0; 45)
   BLOCK@[0; 45)
     L_CURLY@[0; 1) "{"
@@ -1088,8 +1095,12 @@ macro_rules! generate_pattern_iterators {
 "#,
     );
 
-    assert_expansion(MacroKind::Items, &rules, r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#,
-        "fn foo () {}");
+    assert_expansion(
+        MacroKind::Items,
+        &rules,
+        r#"generate_pattern_iterators ! ( double ended ; with # [ stable ( feature = "rust1" , since = "1.0.0" ) ] , Split , RSplit , & 'a str );"#,
+        "fn foo () {}",
+    );
 }
 
 #[test]
@@ -1171,8 +1182,12 @@ fn test_impl_nonzero_fmt() {
 "#,
     );
 
-    assert_expansion(MacroKind::Items, &rules, r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#,
-        "fn foo () {}");
+    assert_expansion(
+        MacroKind::Items,
+        &rules,
+        r#"impl_nonzero_fmt! { # [stable(feature= "nonzero",since="1.28.0")] (Debug,Display,Binary,Octal,LowerHex,UpperHex) for NonZeroU8}"#,
+        "fn foo () {}",
+    );
 }
 
 #[test]
@@ -1189,8 +1204,12 @@ fn test_cfg_if_items() {
 "#,
     );
 
-    assert_expansion(MacroKind::Items, &rules, r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#,
-        "__cfg_if_items ! {(rustdoc ,) ;}");
+    assert_expansion(
+        MacroKind::Items,
+        &rules,
+        r#"__cfg_if_items ! { ( rustdoc , ) ; ( ( ) ( # [ cfg ( any ( target_os = "redox" , unix ) ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as unix ; # [ cfg ( windows ) ] # [ stable ( feature = "rust1" , since = "1.0.0" ) ] pub use sys :: ext as windows ; # [ cfg ( any ( target_os = "linux" , target_os = "l4re" ) ) ] pub mod linux ; ) ) , }"#,
+        "__cfg_if_items ! {(rustdoc ,) ;}",
+    );
 }
 
 #[test]
@@ -1233,10 +1252,13 @@ cfg_if !   {
 "#,
         "__cfg_if_items ! {() ; ((target_env = \"msvc\") ()) , ((all (target_arch = \"wasm32\" , not (target_os = \"emscripten\"))) ()) , (() (mod libunwind ; pub use libunwind :: * ;)) ,}");
 
-    assert_expansion(MacroKind::Items, &rules, r#"
+    assert_expansion(
+        MacroKind::Items,
+        &rules,
+        r#"
 cfg_if ! { @ __apply cfg ( all ( not ( any ( not ( any ( target_os = "solaris" , target_os = "illumos" ) ) ) ) ) ) , }
 "#,
-        ""
+        "",
     );
 }
 
@@ -1291,10 +1313,13 @@ macro_rules! RIDL {
 }"#,
     );
 
-    let expanded = expand(&rules, r#"
+    let expanded = expand(
+        &rules,
+        r#"
 RIDL!{interface ID3D11Asynchronous(ID3D11AsynchronousVtbl): ID3D11DeviceChild(ID3D11DeviceChildVtbl) {
     fn GetDataSize(&mut self) -> UINT
-}}"#);
+}}"#,
+    );
     assert_eq!(expanded.to_string(), "impl ID3D11Asynchronous {pub unsafe fn GetDataSize (& mut self) -> UINT {((* self . lpVtbl) .GetDataSize) (self)}}");
 }
 
@@ -1340,7 +1365,8 @@ quick_error ! (SORT [enum Wrapped # [derive (Debug)]] items [
 
 #[test]
 fn test_empty_repeat_vars_in_empty_repeat_vars() {
-    let rules = create_rules(r#"
+    let rules = create_rules(
+        r#"
 macro_rules! delegate_impl {
     ([$self_type:ident, $self_wrap:ty, $self_map:ident]
      pub trait $name:ident $(: $sup:ident)* $(+ $more_sup:ident)* {
@@ -1385,9 +1411,15 @@ macro_rules! delegate_impl {
         }
     }
 }
-"#);
+"#,
+    );
 
-    assert_expansion(MacroKind::Items, &rules, r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#, "impl <> Data for & \'a mut G where G : Data {}");
+    assert_expansion(
+        MacroKind::Items,
+        &rules,
+        r#"delegate_impl ! {[G , & 'a mut G , deref] pub trait Data : GraphBase {@ section type type NodeWeight ;}}"#,
+        "impl <> Data for & \'a mut G where G : Data {}",
+    );
 }
 
 pub(crate) fn create_rules(macro_definition: &str) -> MacroRules {
@@ -1436,22 +1468,30 @@ pub(crate) fn assert_expansion(
     };
     let (expanded_tree, expected_tree) = match kind {
         MacroKind::Items => {
-            let expanded_tree = token_tree_to_items(&expanded).unwrap().tree();
-            let expected_tree = token_tree_to_items(&expected).unwrap().tree();
+            let expanded_tree =
+                token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node();
+            let expected_tree =
+                token_tree_to_syntax_node(&expected, FragmentKind::Items).unwrap().0.syntax_node();
 
             (
-                debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(),
-                debug_dump_ignore_spaces(expected_tree.syntax()).trim().to_string(),
+                debug_dump_ignore_spaces(&expanded_tree).trim().to_string(),
+                debug_dump_ignore_spaces(&expected_tree).trim().to_string(),
             )
         }
 
         MacroKind::Stmts => {
-            let expanded_tree = token_tree_to_macro_stmts(&expanded).unwrap().tree();
-            let expected_tree = token_tree_to_macro_stmts(&expected).unwrap().tree();
+            let expanded_tree = token_tree_to_syntax_node(&expanded, FragmentKind::Statements)
+                .unwrap()
+                .0
+                .syntax_node();
+            let expected_tree = token_tree_to_syntax_node(&expected, FragmentKind::Statements)
+                .unwrap()
+                .0
+                .syntax_node();
 
             (
-                debug_dump_ignore_spaces(expanded_tree.syntax()).trim().to_string(),
-                debug_dump_ignore_spaces(expected_tree.syntax()).trim().to_string(),
+                debug_dump_ignore_spaces(&expanded_tree).trim().to_string(),
+                debug_dump_ignore_spaces(&expected_tree).trim().to_string(),
             )
         }
     };