6 files changed, 72 insertions, 33 deletions

diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs
index b26441253..62c60e336 100644
--- a/crates/ra_hir_expand/src/builtin_derive.rs
+++ b/crates/ra_hir_expand/src/builtin_derive.rs
@@ -97,11 +97,24 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, mbe::ExpandError> {
 
 fn make_type_args(n: usize, bound: Vec<tt::TokenTree>) -> Vec<tt::TokenTree> {
     let mut result = Vec::<tt::TokenTree>::new();
-    result.push(tt::Leaf::Punct(tt::Punct { char: '<', spacing: tt::Spacing::Alone }).into());
+    result.push(
+        tt::Leaf::Punct(tt::Punct {
+            char: '<',
+            spacing: tt::Spacing::Alone,
+            id: tt::TokenId::unspecified(),
+        })
+        .into(),
+    );
     for i in 0..n {
         if i > 0 {
-            result
-                .push(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone }).into());
+            result.push(
+                tt::Leaf::Punct(tt::Punct {
+                    char: ',',
+                    spacing: tt::Spacing::Alone,
+                    id: tt::TokenId::unspecified(),
+                })
+                .into(),
+            );
         }
         result.push(
             tt::Leaf::Ident(tt::Ident {
@@ -112,7 +125,14 @@ fn make_type_args(n: usize, bound: Vec<tt::TokenTree>) -> Vec<tt::TokenTree> {
         );
         result.extend(bound.iter().cloned());
     }
-    result.push(tt::Leaf::Punct(tt::Punct { char: '>', spacing: tt::Spacing::Alone }).into());
+    result.push(
+        tt::Leaf::Punct(tt::Punct {
+            char: '>',
+            spacing: tt::Spacing::Alone,
+            id: tt::TokenId::unspecified(),
+        })
+        .into(),
+    );
     result
 }
 
diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs
index aa8a5f23f..bce38cc67 100644
--- a/crates/ra_hir_expand/src/quote.rs
+++ b/crates/ra_hir_expand/src/quote.rs
@@ -29,6 +29,7 @@ macro_rules! __quote {
                 tt::Leaf::Punct(tt::Punct {
                     char: $first,
                     spacing: tt::Spacing::Alone,
+                    id: tt::TokenId::unspecified(),
                 }).into()
             ]
         }
@@ -40,10 +41,12 @@ macro_rules! __quote {
                 tt::Leaf::Punct(tt::Punct {
                     char: $first,
                     spacing: tt::Spacing::Joint,
+                    id: tt::TokenId::unspecified(),
                 }).into(),
                 tt::Leaf::Punct(tt::Punct {
                     char: $sec,
                     spacing: tt::Spacing::Alone,
+                    id: tt::TokenId::unspecified(),
                 }).into()
             ]
         }
@@ -179,15 +182,15 @@ macro_rules! impl_to_to_tokentrees {
 }
 
 impl_to_to_tokentrees! {
-    u32 => self { tt::Literal{text: self.to_string().into()} };
-    usize => self { tt::Literal{text: self.to_string().into()}};
-    i32 => self { tt::Literal{text: self.to_string().into()}};
+    u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
+    usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}};
+    i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}};
     tt::Leaf => self { self };
     tt::Literal => self { self };
     tt::Ident => self { self };
     tt::Punct => self { self };
-    &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}};
-    String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}}
+    &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}};
+    String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}}
 }
 
 #[cfg(test)]
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs
index f7636db11..eda66cd50 100644
--- a/crates/ra_mbe/src/mbe_expander/transcriber.rs
+++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs
@@ -108,7 +108,12 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError>
         let tt = tt::Subtree {
             delimiter: None,
             token_trees: vec![
-                tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(),
+                tt::Leaf::from(tt::Punct {
+                    char: '$',
+                    spacing: tt::Spacing::Alone,
+                    id: tt::TokenId::unspecified(),
+                })
+                .into(),
                 tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
                     .into(),
             ],
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index b8e2cfc1d..8f65ff125 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -136,11 +136,15 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
     }
 
     fn mk_punct(c: char) -> tt::TokenTree {
-        tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone }))
+        tt::TokenTree::from(tt::Leaf::from(tt::Punct {
+            char: c,
+            spacing: tt::Spacing::Alone,
+            id: tt::TokenId::unspecified(),
+        }))
     }
 
     fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
-        let lit = tt::Literal { text: doc_comment_text(comment) };
+        let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
 
         tt::TokenTree::from(tt::Leaf::from(lit))
     }
@@ -223,24 +227,29 @@ impl Convertor {
                             .take(token.text().len() - 1)
                             .chain(std::iter::once(last_spacing));
                         for (char, spacing) in token.text().chars().zip(spacing_iter) {
-                            token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into());
+                            let id = self.alloc(token.text_range());
+                            token_trees
+                                .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into());
                         }
                     } else {
-                        let child: tt::TokenTree =
-                            if token.kind() == T![true] || token.kind() == T![false] {
-                                tt::Leaf::from(tt::Literal { text: token.text().clone() }).into()
-                            } else if token.kind().is_keyword()
-                                || token.kind() == IDENT
-                                || token.kind() == LIFETIME
-                            {
-                                let id = self.alloc(token.text_range());
-                                let text = token.text().clone();
-                                tt::Leaf::from(tt::Ident { text, id }).into()
-                            } else if token.kind().is_literal() {
-                                tt::Leaf::from(tt::Literal { text: token.text().clone() }).into()
-                            } else {
-                                return None;
-                            };
+                        let child: tt::TokenTree = if token.kind() == T![true]
+                            || token.kind() == T![false]
+                        {
+                            let id = self.alloc(token.text_range());
+                            tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into()
+                        } else if token.kind().is_keyword()
+                            || token.kind() == IDENT
+                            || token.kind() == LIFETIME
+                        {
+                            let id = self.alloc(token.text_range());
+                            let text = token.text().clone();
+                            tt::Leaf::from(tt::Ident { text, id }).into()
+                        } else if token.kind().is_literal() {
+                            let id = self.alloc(token.text_range());
+                            tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into()
+                        } else {
+                            return None;
+                        };
                         token_trees.push(child);
                     }
                 }
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 148cc2625..70e65bc74 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -78,12 +78,12 @@ macro_rules! foobar {
 
     assert_eq!(expansion.token_trees.len(), 3);
     // ($e:ident) => { foo bar $e }
-    //   0 1            2   3   4
-    assert_eq!(get_id(&expansion.token_trees[0]), Some(2));
-    assert_eq!(get_id(&expansion.token_trees[1]), Some(3));
+    //  0123      45    6   7  89
+    assert_eq!(get_id(&expansion.token_trees[0]), Some(6));
+    assert_eq!(get_id(&expansion.token_trees[1]), Some(7));
 
-    // So baz should be 5
-    assert_eq!(get_id(&expansion.token_trees[2]), Some(5));
+    // So baz should be 10
+    assert_eq!(get_id(&expansion.token_trees[2]), Some(10));
 }
 
 #[test]
diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs
index e7bfd5fd2..209ca4048 100644
--- a/crates/ra_tt/src/lib.rs
+++ b/crates/ra_tt/src/lib.rs
@@ -64,12 +64,14 @@ pub enum Delimiter {
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Literal {
     pub text: SmolStr,
+    pub id: TokenId,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub struct Punct {
     pub char: char,
     pub spacing: Spacing,
+    pub id: TokenId,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]