aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
Diffstat (limited to 'crates')
-rw-r--r--crates/ra_hir_expand/src/builtin_derive.rs28
-rw-r--r--crates/ra_hir_expand/src/lib.rs4
-rw-r--r--crates/ra_hir_expand/src/quote.rs27
-rw-r--r--crates/ra_mbe/src/lib.rs18
-rw-r--r--crates/ra_mbe/src/mbe_expander/matcher.rs2
-rw-r--r--crates/ra_mbe/src/mbe_expander/transcriber.rs7
-rw-r--r--crates/ra_mbe/src/subtree_source.rs12
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs176
-rw-r--r--crates/ra_mbe/src/tests.rs57
-rw-r--r--crates/ra_tt/src/lib.rs22
10 files changed, 272 insertions, 81 deletions
diff --git a/crates/ra_hir_expand/src/builtin_derive.rs b/crates/ra_hir_expand/src/builtin_derive.rs
index b26441253..62c60e336 100644
--- a/crates/ra_hir_expand/src/builtin_derive.rs
+++ b/crates/ra_hir_expand/src/builtin_derive.rs
@@ -97,11 +97,24 @@ fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, mbe::ExpandError> {
97 97
98fn make_type_args(n: usize, bound: Vec<tt::TokenTree>) -> Vec<tt::TokenTree> { 98fn make_type_args(n: usize, bound: Vec<tt::TokenTree>) -> Vec<tt::TokenTree> {
99 let mut result = Vec::<tt::TokenTree>::new(); 99 let mut result = Vec::<tt::TokenTree>::new();
100 result.push(tt::Leaf::Punct(tt::Punct { char: '<', spacing: tt::Spacing::Alone }).into()); 100 result.push(
101 tt::Leaf::Punct(tt::Punct {
102 char: '<',
103 spacing: tt::Spacing::Alone,
104 id: tt::TokenId::unspecified(),
105 })
106 .into(),
107 );
101 for i in 0..n { 108 for i in 0..n {
102 if i > 0 { 109 if i > 0 {
103 result 110 result.push(
104 .push(tt::Leaf::Punct(tt::Punct { char: ',', spacing: tt::Spacing::Alone }).into()); 111 tt::Leaf::Punct(tt::Punct {
112 char: ',',
113 spacing: tt::Spacing::Alone,
114 id: tt::TokenId::unspecified(),
115 })
116 .into(),
117 );
105 } 118 }
106 result.push( 119 result.push(
107 tt::Leaf::Ident(tt::Ident { 120 tt::Leaf::Ident(tt::Ident {
@@ -112,7 +125,14 @@ fn make_type_args(n: usize, bound: Vec<tt::TokenTree>) -> Vec<tt::TokenTree> {
112 ); 125 );
113 result.extend(bound.iter().cloned()); 126 result.extend(bound.iter().cloned());
114 } 127 }
115 result.push(tt::Leaf::Punct(tt::Punct { char: '>', spacing: tt::Spacing::Alone }).into()); 128 result.push(
129 tt::Leaf::Punct(tt::Punct {
130 char: '>',
131 spacing: tt::Spacing::Alone,
132 id: tt::TokenId::unspecified(),
133 })
134 .into(),
135 );
116 result 136 result
117} 137}
118 138
diff --git a/crates/ra_hir_expand/src/lib.rs b/crates/ra_hir_expand/src/lib.rs
index cb4e1950b..2fa5d5140 100644
--- a/crates/ra_hir_expand/src/lib.rs
+++ b/crates/ra_hir_expand/src/lib.rs
@@ -227,7 +227,7 @@ impl ExpansionInfo {
227 let token_id = self.macro_arg.1.token_by_range(range)?; 227 let token_id = self.macro_arg.1.token_by_range(range)?;
228 let token_id = self.macro_def.0.map_id_down(token_id); 228 let token_id = self.macro_def.0.map_id_down(token_id);
229 229
230 let range = self.exp_map.range_by_token(token_id)?; 230 let range = self.exp_map.range_by_token(token_id)?.by_kind(token.value.kind())?;
231 231
232 let token = algo::find_covering_element(&self.expanded.value, range).into_token()?; 232 let token = algo::find_covering_element(&self.expanded.value, range).into_token()?;
233 233
@@ -248,7 +248,7 @@ impl ExpansionInfo {
248 } 248 }
249 }; 249 };
250 250
251 let range = token_map.range_by_token(token_id)?; 251 let range = token_map.range_by_token(token_id)?.by_kind(token.value.kind())?;
252 let token = algo::find_covering_element(&tt.value, range + tt.value.text_range().start()) 252 let token = algo::find_covering_element(&tt.value, range + tt.value.text_range().start())
253 .into_token()?; 253 .into_token()?;
254 Some((tt.with_value(token), origin)) 254 Some((tt.with_value(token), origin))
diff --git a/crates/ra_hir_expand/src/quote.rs b/crates/ra_hir_expand/src/quote.rs
index aa8a5f23f..49155fe62 100644
--- a/crates/ra_hir_expand/src/quote.rs
+++ b/crates/ra_hir_expand/src/quote.rs
@@ -16,7 +16,10 @@ macro_rules! __quote {
16 { 16 {
17 let children = $crate::__quote!($($tt)*); 17 let children = $crate::__quote!($($tt)*);
18 let subtree = tt::Subtree { 18 let subtree = tt::Subtree {
19 delimiter: Some(tt::Delimiter::$delim), 19 delimiter: Some(tt::Delimiter {
20 kind: tt::DelimiterKind::$delim,
21 id: tt::TokenId::unspecified(),
22 }),
20 token_trees: $crate::quote::IntoTt::to_tokens(children), 23 token_trees: $crate::quote::IntoTt::to_tokens(children),
21 }; 24 };
22 subtree 25 subtree
@@ -29,6 +32,7 @@ macro_rules! __quote {
29 tt::Leaf::Punct(tt::Punct { 32 tt::Leaf::Punct(tt::Punct {
30 char: $first, 33 char: $first,
31 spacing: tt::Spacing::Alone, 34 spacing: tt::Spacing::Alone,
35 id: tt::TokenId::unspecified(),
32 }).into() 36 }).into()
33 ] 37 ]
34 } 38 }
@@ -40,10 +44,12 @@ macro_rules! __quote {
40 tt::Leaf::Punct(tt::Punct { 44 tt::Leaf::Punct(tt::Punct {
41 char: $first, 45 char: $first,
42 spacing: tt::Spacing::Joint, 46 spacing: tt::Spacing::Joint,
47 id: tt::TokenId::unspecified(),
43 }).into(), 48 }).into(),
44 tt::Leaf::Punct(tt::Punct { 49 tt::Leaf::Punct(tt::Punct {
45 char: $sec, 50 char: $sec,
46 spacing: tt::Spacing::Alone, 51 spacing: tt::Spacing::Alone,
52 id: tt::TokenId::unspecified(),
47 }).into() 53 }).into()
48 ] 54 ]
49 } 55 }
@@ -179,15 +185,15 @@ macro_rules! impl_to_to_tokentrees {
179} 185}
180 186
181impl_to_to_tokentrees! { 187impl_to_to_tokentrees! {
182 u32 => self { tt::Literal{text: self.to_string().into()} }; 188 u32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()} };
183 usize => self { tt::Literal{text: self.to_string().into()}}; 189 usize => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}};
184 i32 => self { tt::Literal{text: self.to_string().into()}}; 190 i32 => self { tt::Literal{text: self.to_string().into(), id: tt::TokenId::unspecified()}};
185 tt::Leaf => self { self }; 191 tt::Leaf => self { self };
186 tt::Literal => self { self }; 192 tt::Literal => self { self };
187 tt::Ident => self { self }; 193 tt::Ident => self { self };
188 tt::Punct => self { self }; 194 tt::Punct => self { self };
189 &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}}; 195 &str => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}};
190 String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into()}} 196 String => self { tt::Literal{text: format!("{:?}", self.escape_default().to_string()).into(), id: tt::TokenId::unspecified()}}
191} 197}
192 198
193#[cfg(test)] 199#[cfg(test)]
@@ -254,8 +260,13 @@ mod tests {
254 let fields = 260 let fields =
255 fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten(); 261 fields.iter().map(|it| quote!(#it: self.#it.clone(), ).token_trees.clone()).flatten();
256 262
257 let list = 263 let list = tt::Subtree {
258 tt::Subtree { delimiter: Some(tt::Delimiter::Brace), token_trees: fields.collect() }; 264 delimiter: Some(tt::Delimiter {
265 kind: tt::DelimiterKind::Brace,
266 id: tt::TokenId::unspecified(),
267 }),
268 token_trees: fields.collect(),
269 };
259 270
260 let quoted = quote! { 271 let quoted = quote! {
261 impl Clone for #struct_name { 272 impl Clone for #struct_name {
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index ce2deadf6..45dad2d10 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -67,7 +67,15 @@ impl Shift {
67 .token_trees 67 .token_trees
68 .iter() 68 .iter()
69 .filter_map(|tt| match tt { 69 .filter_map(|tt| match tt {
70 tt::TokenTree::Subtree(subtree) => max_id(subtree), 70 tt::TokenTree::Subtree(subtree) => {
71 let tree_id = max_id(subtree);
72 match subtree.delimiter {
73 Some(it) if it.id != tt::TokenId::unspecified() => {
74 Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0)))
75 }
76 _ => tree_id,
77 }
78 }
71 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) 79 tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
72 if ident.id != tt::TokenId::unspecified() => 80 if ident.id != tt::TokenId::unspecified() =>
73 { 81 {
@@ -85,9 +93,13 @@ impl Shift {
85 match t { 93 match t {
86 tt::TokenTree::Leaf(leaf) => match leaf { 94 tt::TokenTree::Leaf(leaf) => match leaf {
87 tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id), 95 tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id),
88 _ => (), 96 tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id),
97 tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id),
89 }, 98 },
90 tt::TokenTree::Subtree(tt) => self.shift_all(tt), 99 tt::TokenTree::Subtree(tt) => {
100 tt.delimiter.as_mut().map(|it: &mut Delimiter| it.id = self.shift(it.id));
101 self.shift_all(tt)
102 }
91 } 103 }
92 } 104 }
93 } 105 }
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 3f5136478..e36b5a412 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -106,7 +106,7 @@ fn match_subtree(
106 } 106 }
107 Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { 107 Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
108 let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; 108 let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
109 if lhs.delimiter != rhs.delimiter { 109 if lhs.delimiter_kind() != rhs.delimiter_kind() {
110 bail!("mismatched delimiter") 110 bail!("mismatched delimiter")
111 } 111 }
112 let mut src = TtIter::new(rhs); 112 let mut src = TtIter::new(rhs);
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs
index f7636db11..eda66cd50 100644
--- a/crates/ra_mbe/src/mbe_expander/transcriber.rs
+++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs
@@ -108,7 +108,12 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError>
108 let tt = tt::Subtree { 108 let tt = tt::Subtree {
109 delimiter: None, 109 delimiter: None,
110 token_trees: vec![ 110 token_trees: vec![
111 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), 111 tt::Leaf::from(tt::Punct {
112 char: '$',
113 spacing: tt::Spacing::Alone,
114 id: tt::TokenId::unspecified(),
115 })
116 .into(),
112 tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) 117 tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
113 .into(), 118 .into(),
114 ], 119 ],
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 061e9f20b..b841c39d3 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -70,11 +70,11 @@ impl<'a> SubtreeTokenSource<'a> {
70 } 70 }
71 Some(tt::TokenTree::Subtree(subtree)) => { 71 Some(tt::TokenTree::Subtree(subtree)) => {
72 self.cached_cursor.set(cursor.subtree().unwrap()); 72 self.cached_cursor.set(cursor.subtree().unwrap());
73 cached.push(Some(convert_delim(subtree.delimiter, false))); 73 cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
74 } 74 }
75 None => { 75 None => {
76 if let Some(subtree) = cursor.end() { 76 if let Some(subtree) = cursor.end() {
77 cached.push(Some(convert_delim(subtree.delimiter, true))); 77 cached.push(Some(convert_delim(subtree.delimiter_kind(), true)));
78 self.cached_cursor.set(cursor.bump()); 78 self.cached_cursor.set(cursor.bump());
79 } 79 }
80 } 80 }
@@ -114,11 +114,11 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
114 } 114 }
115} 115}
116 116
117fn convert_delim(d: Option<tt::Delimiter>, closing: bool) -> TtToken { 117fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
118 let (kinds, texts) = match d { 118 let (kinds, texts) = match d {
119 Some(tt::Delimiter::Parenthesis) => ([T!['('], T![')']], "()"), 119 Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"),
120 Some(tt::Delimiter::Brace) => ([T!['{'], T!['}']], "{}"), 120 Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"),
121 Some(tt::Delimiter::Bracket) => ([T!['['], T![']']], "[]"), 121 Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"),
122 None => ([L_DOLLAR, R_DOLLAR], ""), 122 None => ([L_DOLLAR, R_DOLLAR], ""),
123 }; 123 };
124 124
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index b8e2cfc1d..2c60430d1 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -5,17 +5,37 @@ use ra_syntax::{
5 ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode, 5 ast, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, SyntaxKind::*, SyntaxNode,
6 SyntaxTreeBuilder, TextRange, TextUnit, T, 6 SyntaxTreeBuilder, TextRange, TextUnit, T,
7}; 7};
8use rustc_hash::FxHashMap;
8use std::iter::successors; 9use std::iter::successors;
9use tt::buffer::{Cursor, TokenBuffer}; 10use tt::buffer::{Cursor, TokenBuffer};
10 11
11use crate::subtree_source::SubtreeTokenSource; 12use crate::subtree_source::SubtreeTokenSource;
12use crate::ExpandError; 13use crate::ExpandError;
13 14
15#[derive(Debug, PartialEq, Eq, Clone, Copy)]
16pub enum TokenTextRange {
17 Token(TextRange),
18 Delimiter(TextRange, TextRange),
19}
20
21impl TokenTextRange {
22 pub fn by_kind(self, kind: SyntaxKind) -> Option<TextRange> {
23 match self {
24 TokenTextRange::Token(it) => Some(it),
25 TokenTextRange::Delimiter(open, close) => match kind {
26 T!['{'] | T!['('] | T!['['] => Some(open),
27 T!['}'] | T![')'] | T![']'] => Some(close),
28 _ => None,
29 },
30 }
31 }
32}
33
14/// Maps `tt::TokenId` to the relative range of the original token. 34/// Maps `tt::TokenId` to the relative range of the original token.
15#[derive(Debug, PartialEq, Eq, Default)] 35#[derive(Debug, PartialEq, Eq, Default)]
16pub struct TokenMap { 36pub struct TokenMap {
17 /// Maps `tt::TokenId` to the *relative* source range. 37 /// Maps `tt::TokenId` to the *relative* source range.
18 entries: Vec<(tt::TokenId, TextRange)>, 38 entries: Vec<(tt::TokenId, TokenTextRange)>,
19} 39}
20 40
21/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro 41/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro
@@ -71,17 +91,32 @@ pub fn token_tree_to_syntax_node(
71 91
72impl TokenMap { 92impl TokenMap {
73 pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> { 93 pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
74 let &(token_id, _) = self.entries.iter().find(|(_, range)| *range == relative_range)?; 94 let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {
95 TokenTextRange::Token(it) => *it == relative_range,
96 TokenTextRange::Delimiter(open, close) => {
97 *open == relative_range || *close == relative_range
98 }
99 })?;
75 Some(token_id) 100 Some(token_id)
76 } 101 }
77 102
78 pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TextRange> { 103 pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TokenTextRange> {
79 let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?; 104 let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?;
80 Some(range) 105 Some(range)
81 } 106 }
82 107
83 fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { 108 fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
84 self.entries.push((token_id, relative_range)); 109 self.entries.push((token_id, TokenTextRange::Token(relative_range)));
110 }
111
112 fn insert_delim(
113 &mut self,
114 token_id: tt::TokenId,
115 open_relative_range: TextRange,
116 close_relative_range: TextRange,
117 ) {
118 self.entries
119 .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range)));
85 } 120 }
86} 121}
87 122
@@ -121,7 +156,10 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
121 token_trees.push(mk_punct('!')); 156 token_trees.push(mk_punct('!'));
122 } 157 }
123 token_trees.push(tt::TokenTree::from(tt::Subtree { 158 token_trees.push(tt::TokenTree::from(tt::Subtree {
124 delimiter: Some(tt::Delimiter::Bracket), 159 delimiter: Some(tt::Delimiter {
160 kind: tt::DelimiterKind::Bracket,
161 id: tt::TokenId::unspecified(),
162 }),
125 token_trees: meta_tkns, 163 token_trees: meta_tkns,
126 })); 164 }));
127 165
@@ -136,11 +174,15 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr
136 } 174 }
137 175
138 fn mk_punct(c: char) -> tt::TokenTree { 176 fn mk_punct(c: char) -> tt::TokenTree {
139 tt::TokenTree::from(tt::Leaf::from(tt::Punct { char: c, spacing: tt::Spacing::Alone })) 177 tt::TokenTree::from(tt::Leaf::from(tt::Punct {
178 char: c,
179 spacing: tt::Spacing::Alone,
180 id: tt::TokenId::unspecified(),
181 }))
140 } 182 }
141 183
142 fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree { 184 fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
143 let lit = tt::Literal { text: doc_comment_text(comment) }; 185 let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
144 186
145 tt::TokenTree::from(tt::Leaf::from(lit)) 187 tt::TokenTree::from(tt::Leaf::from(lit))
146 } 188 }
@@ -186,12 +228,16 @@ impl Convertor {
186 .last() 228 .last()
187 .unwrap(); 229 .unwrap();
188 230
189 let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { 231 let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) {
190 (T!['('], T![')']) => (Some(tt::Delimiter::Parenthesis), true), 232 (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true),
191 (T!['{'], T!['}']) => (Some(tt::Delimiter::Brace), true), 233 (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true),
192 (T!['['], T![']']) => (Some(tt::Delimiter::Bracket), true), 234 (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true),
193 _ => (None, false), 235 _ => (None, false),
194 }; 236 };
237 let delimiter = delimiter_kind.map(|kind| tt::Delimiter {
238 kind,
239 id: self.alloc_delim(first_child.text_range(), last_child.text_range()),
240 });
195 241
196 let mut token_trees = Vec::new(); 242 let mut token_trees = Vec::new();
197 let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); 243 let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable();
@@ -223,25 +269,34 @@ impl Convertor {
223 .take(token.text().len() - 1) 269 .take(token.text().len() - 1)
224 .chain(std::iter::once(last_spacing)); 270 .chain(std::iter::once(last_spacing));
225 for (char, spacing) in token.text().chars().zip(spacing_iter) { 271 for (char, spacing) in token.text().chars().zip(spacing_iter) {
226 token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); 272 token_trees.push(
273 tt::Leaf::from(tt::Punct {
274 char,
275 spacing,
276 id: self.alloc(token.text_range()),
277 })
278 .into(),
279 );
227 } 280 }
228 } else { 281 } else {
229 let child: tt::TokenTree = 282 macro_rules! make_leaf {
230 if token.kind() == T![true] || token.kind() == T![false] { 283 ($i:ident) => {
231 tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() 284 tt::$i {
232 } else if token.kind().is_keyword() 285 id: self.alloc(token.text_range()),
233 || token.kind() == IDENT 286 text: token.text().clone(),
234 || token.kind() == LIFETIME 287 }
235 { 288 .into()
236 let id = self.alloc(token.text_range());
237 let text = token.text().clone();
238 tt::Leaf::from(tt::Ident { text, id }).into()
239 } else if token.kind().is_literal() {
240 tt::Leaf::from(tt::Literal { text: token.text().clone() }).into()
241 } else {
242 return None;
243 }; 289 };
244 token_trees.push(child); 290 }
291
292 let child: tt::Leaf = match token.kind() {
293 T![true] | T![false] => make_leaf!(Literal),
294 IDENT | LIFETIME => make_leaf!(Ident),
295 k if k.is_keyword() => make_leaf!(Ident),
296 k if k.is_literal() => make_leaf!(Literal),
297 _ => return None,
298 };
299 token_trees.push(child.into());
245 } 300 }
246 } 301 }
247 NodeOrToken::Node(node) => { 302 NodeOrToken::Node(node) => {
@@ -266,11 +321,26 @@ impl Convertor {
266 self.map.insert(token_id, relative_range); 321 self.map.insert(token_id, relative_range);
267 token_id 322 token_id
268 } 323 }
324
325 fn alloc_delim(
326 &mut self,
327 open_abs_range: TextRange,
328 close_abs_range: TextRange,
329 ) -> tt::TokenId {
330 let open_relative_range = open_abs_range - self.global_offset;
331 let close_relative_range = close_abs_range - self.global_offset;
332 let token_id = tt::TokenId(self.next_id);
333 self.next_id += 1;
334
335 self.map.insert_delim(token_id, open_relative_range, close_relative_range);
336 token_id
337 }
269} 338}
270 339
271struct TtTreeSink<'a> { 340struct TtTreeSink<'a> {
272 buf: String, 341 buf: String,
273 cursor: Cursor<'a>, 342 cursor: Cursor<'a>,
343 open_delims: FxHashMap<tt::TokenId, TextUnit>,
274 text_pos: TextUnit, 344 text_pos: TextUnit,
275 inner: SyntaxTreeBuilder, 345 inner: SyntaxTreeBuilder,
276 token_map: TokenMap, 346 token_map: TokenMap,
@@ -285,6 +355,7 @@ impl<'a> TtTreeSink<'a> {
285 TtTreeSink { 355 TtTreeSink {
286 buf: String::new(), 356 buf: String::new(),
287 cursor, 357 cursor,
358 open_delims: FxHashMap::default(),
288 text_pos: 0.into(), 359 text_pos: 0.into(),
289 inner: SyntaxTreeBuilder::default(), 360 inner: SyntaxTreeBuilder::default(),
290 roots: smallvec::SmallVec::new(), 361 roots: smallvec::SmallVec::new(),
@@ -297,11 +368,11 @@ impl<'a> TtTreeSink<'a> {
297 } 368 }
298} 369}
299 370
300fn delim_to_str(d: Option<tt::Delimiter>, closing: bool) -> SmolStr { 371fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> SmolStr {
301 let texts = match d { 372 let texts = match d {
302 Some(tt::Delimiter::Parenthesis) => "()", 373 Some(tt::DelimiterKind::Parenthesis) => "()",
303 Some(tt::Delimiter::Brace) => "{}", 374 Some(tt::DelimiterKind::Brace) => "{}",
304 Some(tt::Delimiter::Bracket) => "[]", 375 Some(tt::DelimiterKind::Bracket) => "[]",
305 None => return "".into(), 376 None => return "".into(),
306 }; 377 };
307 378
@@ -322,34 +393,49 @@ impl<'a> TreeSink for TtTreeSink<'a> {
322 break; 393 break;
323 } 394 }
324 395
325 match self.cursor.token_tree() { 396 let text: SmolStr = match self.cursor.token_tree() {
326 Some(tt::TokenTree::Leaf(leaf)) => { 397 Some(tt::TokenTree::Leaf(leaf)) => {
327 // Mark the range if needed 398 // Mark the range if needed
328 if let tt::Leaf::Ident(ident) = leaf { 399 let id = match leaf {
329 if kind == IDENT { 400 tt::Leaf::Ident(ident) => ident.id,
330 let range = 401 tt::Leaf::Punct(punct) => punct.id,
331 TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); 402 tt::Leaf::Literal(lit) => lit.id,
332 self.token_map.insert(ident.id, range); 403 };
333 } 404 let text = SmolStr::new(format!("{}", leaf));
334 } 405 let range = TextRange::offset_len(self.text_pos, TextUnit::of_str(&text));
335 406 self.token_map.insert(id, range);
336 self.cursor = self.cursor.bump(); 407 self.cursor = self.cursor.bump();
337 self.buf += &format!("{}", leaf); 408 text
338 } 409 }
339 Some(tt::TokenTree::Subtree(subtree)) => { 410 Some(tt::TokenTree::Subtree(subtree)) => {
340 self.cursor = self.cursor.subtree().unwrap(); 411 self.cursor = self.cursor.subtree().unwrap();
341 self.buf += &delim_to_str(subtree.delimiter, false); 412 if let Some(id) = subtree.delimiter.map(|it| it.id) {
413 self.open_delims.insert(id, self.text_pos);
414 }
415 delim_to_str(subtree.delimiter_kind(), false)
342 } 416 }
343 None => { 417 None => {
344 if let Some(parent) = self.cursor.end() { 418 if let Some(parent) = self.cursor.end() {
345 self.cursor = self.cursor.bump(); 419 self.cursor = self.cursor.bump();
346 self.buf += &delim_to_str(parent.delimiter, true); 420 if let Some(id) = parent.delimiter.map(|it| it.id) {
421 if let Some(open_delim) = self.open_delims.get(&id) {
422 let open_range =
423 TextRange::offset_len(*open_delim, TextUnit::from_usize(1));
424 let close_range =
425 TextRange::offset_len(self.text_pos, TextUnit::from_usize(1));
426 self.token_map.insert_delim(id, open_range, close_range);
427 }
428 }
429 delim_to_str(parent.delimiter_kind(), true)
430 } else {
431 continue;
347 } 432 }
348 } 433 }
349 }; 434 };
435 self.buf += &text;
436 self.text_pos += TextUnit::of_str(&text);
350 } 437 }
351 438
352 self.text_pos += TextUnit::of_str(&self.buf);
353 let text = SmolStr::new(self.buf.as_str()); 439 let text = SmolStr::new(self.buf.as_str());
354 self.buf.clear(); 440 self.buf.clear();
355 self.inner.token(kind, text); 441 self.inner.token(kind, text);
@@ -495,7 +581,7 @@ mod tests {
495 let token_tree = ast::TokenTree::cast(token_tree).unwrap(); 581 let token_tree = ast::TokenTree::cast(token_tree).unwrap();
496 let tt = ast_to_token_tree(&token_tree).unwrap().0; 582 let tt = ast_to_token_tree(&token_tree).unwrap().0;
497 583
498 assert_eq!(tt.delimiter, Some(tt::Delimiter::Brace)); 584 assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace));
499 } 585 }
500 586
501 #[test] 587 #[test]
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 148cc2625..ff225f0db 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -77,13 +77,41 @@ macro_rules! foobar {
77 } 77 }
78 78
79 assert_eq!(expansion.token_trees.len(), 3); 79 assert_eq!(expansion.token_trees.len(), 3);
80 // ($e:ident) => { foo bar $e } 80 // {($e:ident) => { foo bar $e }}
81 // 0 1 2 3 4 81 // 012345 67 8 9 T 12
82 assert_eq!(get_id(&expansion.token_trees[0]), Some(2)); 82 assert_eq!(get_id(&expansion.token_trees[0]), Some(9));
83 assert_eq!(get_id(&expansion.token_trees[1]), Some(3)); 83 assert_eq!(get_id(&expansion.token_trees[1]), Some(10));
84 84
85 // So baz should be 5 85 // The input args of macro call include parentheses:
86 assert_eq!(get_id(&expansion.token_trees[2]), Some(5)); 86 // (baz)
87 // So baz should be 12+1+1
88 assert_eq!(get_id(&expansion.token_trees[2]), Some(14));
89}
90
91#[test]
92fn test_token_map() {
93 use ra_parser::SyntaxKind::*;
94 use ra_syntax::T;
95
96 let macro_definition = r#"
97macro_rules! foobar {
98 ($e:ident) => { fn $e() {} }
99}
100"#;
101 let rules = create_rules(macro_definition);
102 let (expansion, (token_map, content)) = expand_and_map(&rules, "foobar!(baz);");
103
104 let get_text = |id, kind| -> String {
105 content[token_map.range_by_token(id).unwrap().by_kind(kind).unwrap()].to_string()
106 };
107
108 assert_eq!(expansion.token_trees.len(), 4);
109 // {($e:ident) => { fn $e() {} }}
110 // 012345 67 8 9 T12 3
111
112 assert_eq!(get_text(tt::TokenId(9), IDENT), "fn");
113 assert_eq!(get_text(tt::TokenId(12), T!['(']), "(");
114 assert_eq!(get_text(tt::TokenId(13), T!['{']), "{");
87} 115}
88 116
89#[test] 117#[test]
@@ -1441,6 +1469,23 @@ pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
1441 rules.expand(&invocation_tt).unwrap() 1469 rules.expand(&invocation_tt).unwrap()
1442} 1470}
1443 1471
1472pub(crate) fn expand_and_map(
1473 rules: &MacroRules,
1474 invocation: &str,
1475) -> (tt::Subtree, (TokenMap, String)) {
1476 let source_file = ast::SourceFile::parse(invocation).ok().unwrap();
1477 let macro_invocation =
1478 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
1479
1480 let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap();
1481 let expanded = rules.expand(&invocation_tt).unwrap();
1482
1483 let (node, expanded_token_tree) =
1484 token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap();
1485
1486 (expanded, (expanded_token_tree, node.syntax_node().to_string()))
1487}
1488
1444pub(crate) enum MacroKind { 1489pub(crate) enum MacroKind {
1445 Items, 1490 Items,
1446 Stmts, 1491 Stmts,
diff --git a/crates/ra_tt/src/lib.rs b/crates/ra_tt/src/lib.rs
index e7bfd5fd2..10f424aae 100644
--- a/crates/ra_tt/src/lib.rs
+++ b/crates/ra_tt/src/lib.rs
@@ -55,7 +55,13 @@ pub struct Subtree {
55} 55}
56 56
57#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] 57#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
58pub enum Delimiter { 58pub struct Delimiter {
59 pub id: TokenId,
60 pub kind: DelimiterKind,
61}
62
63#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
64pub enum DelimiterKind {
59 Parenthesis, 65 Parenthesis,
60 Brace, 66 Brace,
61 Bracket, 67 Bracket,
@@ -64,12 +70,14 @@ pub enum Delimiter {
64#[derive(Debug, Clone, PartialEq, Eq, Hash)] 70#[derive(Debug, Clone, PartialEq, Eq, Hash)]
65pub struct Literal { 71pub struct Literal {
66 pub text: SmolStr, 72 pub text: SmolStr,
73 pub id: TokenId,
67} 74}
68 75
69#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 76#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
70pub struct Punct { 77pub struct Punct {
71 pub char: char, 78 pub char: char,
72 pub spacing: Spacing, 79 pub spacing: Spacing,
80 pub id: TokenId,
73} 81}
74 82
75#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] 83#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
@@ -95,10 +103,10 @@ impl fmt::Display for TokenTree {
95 103
96impl fmt::Display for Subtree { 104impl fmt::Display for Subtree {
97 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 105 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
98 let (l, r) = match self.delimiter { 106 let (l, r) = match self.delimiter_kind() {
99 Some(Delimiter::Parenthesis) => ("(", ")"), 107 Some(DelimiterKind::Parenthesis) => ("(", ")"),
100 Some(Delimiter::Brace) => ("{", "}"), 108 Some(DelimiterKind::Brace) => ("{", "}"),
101 Some(Delimiter::Bracket) => ("[", "]"), 109 Some(DelimiterKind::Bracket) => ("[", "]"),
102 None => ("", ""), 110 None => ("", ""),
103 }; 111 };
104 f.write_str(l)?; 112 f.write_str(l)?;
@@ -163,6 +171,10 @@ impl Subtree {
163 171
164 self.token_trees.len() + children_count 172 self.token_trees.len() + children_count
165 } 173 }
174
175 pub fn delimiter_kind(&self) -> Option<DelimiterKind> {
176 self.delimiter.map(|it| it.kind)
177 }
166} 178}
167 179
168pub mod buffer; 180pub mod buffer;