Add token id to delims

6 files changed, 135 insertions, 65 deletions

diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs
index bce38cc67..49155fe62 100644
--- a/crates/ra_hir_expand/src/quote.rs
+++ b/crates/ra_hir_expand/src/quote.rs
@@ -16,7 +16,10 @@ macro_rules! __quote {
         {
             let children = $crate::__quote!($($tt)*);
             let subtree = tt::Subtree {
-                delimiter: Some(tt::Delimiter::$delim),
+                delimiter: Some(tt::Delimiter {
+                    kind: tt::DelimiterKind::$delim,
+                    id: tt::TokenId::unspecified(),
+                }),
                 token_trees: $crate::quote::IntoTt::to_tokens(children),
             };
             subtree
@@ -257,8 +260,13 @@ mod tests {
         let fields =
             fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten();
 
-        let list =
-            tt::Subtree { delimiter: Some(tt::Delimiter::Brace), token_trees: fields.collect() };
+        let list = tt::Subtree {
+            delimiter: Some(tt::Delimiter {
+                kind: tt::DelimiterKind::Brace,
+                id: tt::TokenId::unspecified(),
+            }),
+            token_trees: fields.collect(),
+        };
 
         let quoted = quote! {
             impl Clone for #struct_name {
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 3f5136478..c67ae4110 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -106,7 +106,7 @@ fn match_subtree(
             }
             Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
                 let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
-                if lhs.delimiter != rhs.delimiter {
+                if lhs.delimiter.map(|it| it.kind) != rhs.delimiter.map(|it| it.kind) {
                     bail!("mismatched delimiter")
                 }
                 let mut src = TtIter::new(rhs);
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 061e9f20b..5a03a372a 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -115,10 +115,10 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
 }
 
 fn convert_delim(d: Option<tt::Delimiter>, closing: bool) -> TtToken {
-    let (kinds, texts) = match d {
-        Some(tt::Delimiter::Parenthesis) => ([T!['('], T![')']], "()"),
-        Some(tt::Delimiter::Brace) => ([T!['{'], T!['}']], "{}"),
-        Some(tt::Delimiter::Bracket) => ([T!['['], T![']']], "[]"),
+    let (kinds, texts) = match d.map(|it| it.kind) {
+        Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"),
+        Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"),
+        Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"),
         None => ([L_DOLLAR, R_DOLLAR], ""),
     };
 
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 8f65ff125..a85bb058b 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -5,6 +5,7 @@ use ra_syntax::{
     ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
     SyntaxTreeBuilder, TextRange, TextUnit, T,
 };
+use rustc_hash::FxHashMap;
 use std::iter::successors;
 use tt::buffer::{Cursor, TokenBuffer};
 
@@ -83,6 +84,15 @@ impl TokenMap {
     fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
         self.entries.push((token_id, relative_range));
     }
+
+    fn insert_delim(
+        &mut self,
+        _token_id: tt::TokenId,
+        _open_relative_range: TextRange,
+        _close_relative_range: TextRange,
+    ) {
+        // FIXME: Add entries for delimiter
+    }
 }
 
 /// Returns the textual content of a doc comment block as a quoted string
@@ -121,7 +131,10 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
         token_trees.push(mk_punct('!'));
     }
     token_trees.push(tt::TokenTree::from(tt::Subtree {
-        delimiter: Some(tt::Delimiter::Bracket),
+        delimiter: Some(tt::Delimiter {
+            kind: tt::DelimiterKind::Bracket,
+            id: tt::TokenId::unspecified(),
+        }),
         token_trees: meta_tkns,
     }));
 
@@ -190,12 +203,16 @@ impl Convertor {
         .last()
         .unwrap();
 
-        let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) {
-            (T!['('], T![')']) => (Some(tt::Delimiter::Parenthesis), true),
-            (T!['{'], T!['}']) => (Some(tt::Delimiter::Brace), true),
-            (T!['['], T![']']) => (Some(tt::Delimiter::Bracket), true),
+        let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) {
+            (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true),
+            (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true),
+            (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true),
             _ => (None, false),
         };
+        let delimiter = delimiter_kind.map(|kind| tt::Delimiter {
+            kind,
+            id: self.alloc_delim(first_child.text_range(), last_child.text_range()),
+        });
 
         let mut token_trees = Vec::new();
         let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
@@ -232,25 +249,31 @@ impl Convertor {
                                 .push(tt::Leaf::from(tt::Punct { char, spacing, id }).into());
                         }
                     } else {
-                        let child: tt::TokenTree = if token.kind() == T![true]
-                            || token.kind() == T![false]
-                        {
-                            let id = self.alloc(token.text_range());
-                            tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into()
-                        } else if token.kind().is_keyword()
-                            || token.kind() == IDENT
-                            || token.kind() == LIFETIME
-                        {
-                            let id = self.alloc(token.text_range());
-                            let text = token.text().clone();
-                            tt::Leaf::from(tt::Ident { text, id }).into()
-                        } else if token.kind().is_literal() {
-                            let id = self.alloc(token.text_range());
-                            tt::Leaf::from(tt::Literal { text: token.text().clone(), id }).into()
-                        } else {
-                            return None;
+                        let child: tt::Leaf = match token.kind() {
+                            T![true] | T![false] => {
+                                let id = self.alloc(token.text_range());
+                                let text = token.text().clone();
+                                tt::Literal { text, id }.into()
+                            }
+                            IDENT | LIFETIME => {
+                                let id = self.alloc(token.text_range());
+                                let text = token.text().clone();
+                                tt::Ident { text, id }.into()
+                            }
+                            k if k.is_keyword() => {
+                                let id = self.alloc(token.text_range());
+                                let text = token.text().clone();
+                                tt::Ident { text, id }.into()
+                            }
+                            k if k.is_literal() => {
+                                let id = self.alloc(token.text_range());
+                                let text = token.text().clone();
+                                tt::Literal { text, id }.into()
+                            }
+                            _ => return None,
                         };
-                        token_trees.push(child);
+
+                        token_trees.push(child.into());
                     }
                 }
                 NodeOrToken::Node(node) => {
@@ -275,11 +298,26 @@ impl Convertor {
         self.map.insert(token_id, relative_range);
         token_id
     }
+
+    fn alloc_delim(
+        &mut self,
+        open_abs_range: TextRange,
+        close_abs_range: TextRange,
+    ) -> tt::TokenId {
+        let open_relative_range = open_abs_range - self.global_offset;
+        let close_relative_range = close_abs_range - self.global_offset;
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+
+        self.map.insert_delim(token_id, open_relative_range, close_relative_range);
+        token_id
+    }
 }
 
 struct TtTreeSink<'a> {
     buf: String,
     cursor: Cursor<'a>,
+    open_delims: FxHashMap<tt::TokenId, TextUnit>,
     text_pos: TextUnit,
     inner: SyntaxTreeBuilder,
     token_map: TokenMap,
@@ -294,6 +332,7 @@ impl<'a> TtTreeSink<'a> {
         TtTreeSink {
             buf: String::new(),
             cursor,
+            open_delims: FxHashMap::default(),
             text_pos: 0.into(),
             inner: SyntaxTreeBuilder::default(),
             roots: smallvec::SmallVec::new(),
@@ -307,10 +346,10 @@ impl<'a> TtTreeSink<'a> {
 }
 
 fn delim_to_str(d: Option<tt::Delimiter>, closing: bool) -> SmolStr {
-    let texts = match d {
-        Some(tt::Delimiter::Parenthesis) => "()",
-        Some(tt::Delimiter::Brace) => "{}",
-        Some(tt::Delimiter::Bracket) => "[]",
+    let texts = match d.map(|it| it.kind) {
+        Some(tt::DelimiterKind::Parenthesis) => "()",
+        Some(tt::DelimiterKind::Brace) => "{}",
+        Some(tt::DelimiterKind::Bracket) => "[]",
         None => return "".into(),
     };
 
@@ -331,34 +370,49 @@ impl<'a> TreeSink for TtTreeSink<'a> {
                 break;
             }
 
-            match self.cursor.token_tree() {
+            let text: Option<SmolStr> = match self.cursor.token_tree() {
                 Some(tt::TokenTree::Leaf(leaf)) => {
                     // Mark the range if needed
-                    if let tt::Leaf::Ident(ident) = leaf {
-                        if kind == IDENT {
-                            let range =
-                                TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text));
-                            self.token_map.insert(ident.id, range);
-                        }
-                    }
-
+                    let id = match leaf {
+                        tt::Leaf::Ident(ident) => ident.id,
+                        tt::Leaf::Punct(punct) => punct.id,
+                        tt::Leaf::Literal(lit) => lit.id,
+                    };
+                    let text = SmolStr::new(format!("{}", leaf));
+                    let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text));
+                    self.token_map.insert(id, range);
                     self.cursor = self.cursor.bump();
-                    self.buf += &format!("{}", leaf);
+                    Some(text)
                 }
                 Some(tt::TokenTree::Subtree(subtree)) => {
                     self.cursor = self.cursor.subtree().unwrap();
-                    self.buf += &delim_to_str(subtree.delimiter, false);
-                }
-                None => {
-                    if let Some(parent) = self.cursor.end() {
-                        self.cursor = self.cursor.bump();
-                        self.buf += &delim_to_str(parent.delimiter, true);
+                    if let Some(id) = subtree.delimiter.map(|it| it.id) {
+                        self.open_delims.insert(id, self.text_pos);
                     }
+                    Some(delim_to_str(subtree.delimiter, false))
                 }
+                None => self.cursor.end().and_then(|parent| {
+                    self.cursor = self.cursor.bump();
+                    if let Some(id) = parent.delimiter.map(|it| it.id) {
+                        if let Some(open_delim) = self.open_delims.get(&id) {
+                            let open_range =
+                                TextRange::offset_len(*open_delim, TextUnit::from_usize(1));
+                            let close_range =
+                                TextRange::offset_len(self.text_pos, TextUnit::from_usize(1));
+                            self.token_map.insert_delim(id, open_range, close_range);
+                        }
+                    }
+
+                    Some(delim_to_str(parent.delimiter, true))
+                }),
             };
+
+            if let Some(text) = text {
+                self.buf += &text;
+                self.text_pos += TextUnit::of_str(&text);
+            }
         }
 
-        self.text_pos += TextUnit::of_str(&self.buf);
         let text = SmolStr::new(self.buf.as_str());
         self.buf.clear();
         self.inner.token(kind, text);
@@ -504,7 +558,7 @@ mod tests {
         let token_tree = ast::TokenTree::cast(token_tree).unwrap();
         let tt = ast_to_token_tree(&token_tree).unwrap().0;
 
-        assert_eq!(tt.delimiter, Some(tt::Delimiter::Brace));
+        assert_eq!(tt.delimiter.map(|it| it.kind), Some(tt::DelimiterKind::Brace));
     }
 
     #[test]
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 70e65bc74..6bcfedcac 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -77,13 +77,15 @@ macro_rules! foobar {
     }
 
     assert_eq!(expansion.token_trees.len(), 3);
-    // ($e:ident) => { foo bar $e }
-    //  0123      45    6   7  89
-    assert_eq!(get_id(&expansion.token_trees[0]), Some(6));
-    assert_eq!(get_id(&expansion.token_trees[1]), Some(7));
-
-    // So baz should be 10
-    assert_eq!(get_id(&expansion.token_trees[2]), Some(10));
+    // {($e:ident) => { foo bar $e }}
+    // 012345      67 8 9   T   12
+    assert_eq!(get_id(&expansion.token_trees[0]), Some(9));
+    assert_eq!(get_id(&expansion.token_trees[1]), Some(10));
+
+    // The input args of macro call include parentheses:
+    // (baz)
+    // So baz should be 12+1+1
+    assert_eq!(get_id(&expansion.token_trees[2]), Some(14));
 }
 
 #[test]
diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs
index 209ca4048..73d8395a8 100644
--- a/crates/ra_tt/src/lib.rs
+++ b/crates/ra_tt/src/lib.rs
@@ -55,7 +55,13 @@ pub struct Subtree {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub enum Delimiter {
+pub struct Delimiter {
+    pub id: TokenId,
+    pub kind: DelimiterKind,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub enum DelimiterKind {
     Parenthesis,
     Brace,
     Bracket,
@@ -97,10 +103,10 @@ impl fmt::Display for TokenTree {
 
 impl fmt::Display for Subtree {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let (l, r) = match self.delimiter {
-            Some(Delimiter::Parenthesis) => ("(", ")"),
-            Some(Delimiter::Brace) => ("{", "}"),
-            Some(Delimiter::Bracket) => ("[", "]"),
+        let (l, r) = match self.delimiter.map(|it| it.kind) {
+            Some(DelimiterKind::Parenthesis) => ("(", ")"),
+            Some(DelimiterKind::Brace) => ("{", "}"),
+            Some(DelimiterKind::Bracket) => ("[", "]"),
             None => ("", ""),
         };
         f.write_str(l)?;