diff options
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r-- | crates/ra_mbe/src/lib.rs | 2 | ||||
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 206 |
2 files changed, 100 insertions, 108 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 58ca95368..bbddebe67 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs | |||
@@ -31,7 +31,7 @@ pub enum ExpandError { | |||
31 | } | 31 | } |
32 | 32 | ||
33 | pub use crate::syntax_bridge::{ | 33 | pub use crate::syntax_bridge::{ |
34 | ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, RevTokenMap, TokenMap, | 34 | ast_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, TokenMap, |
35 | }; | 35 | }; |
36 | 36 | ||
37 | /// This struct contains AST for a single `macro_rules` definition. What might | 37 | /// This struct contains AST for a single `macro_rules` definition. What might |
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 8398c9ac7..d1c49c0b3 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -14,30 +14,22 @@ use crate::ExpandError; | |||
14 | #[derive(Debug, PartialEq, Eq, Default)] | 14 | #[derive(Debug, PartialEq, Eq, Default)] |
15 | pub struct TokenMap { | 15 | pub struct TokenMap { |
16 | /// Maps `tt::TokenId` to the *relative* source range. | 16 | /// Maps `tt::TokenId` to the *relative* source range. |
17 | tokens: Vec<TextRange>, | 17 | entries: Vec<(tt::TokenId, TextRange)>, |
18 | } | ||
19 | |||
20 | /// Maps relative range of the expanded syntax node to `tt::TokenId` | ||
21 | #[derive(Debug, PartialEq, Eq, Default)] | ||
22 | pub struct RevTokenMap { | ||
23 | pub ranges: Vec<(TextRange, tt::TokenId)>, | ||
24 | } | 18 | } |
25 | 19 | ||
26 | /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro | 20 | /// Convert the syntax tree (what user has written) to a `TokenTree` (what macro |
27 | /// will consume). | 21 | /// will consume). |
28 | pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { | 22 | pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)> { |
29 | let mut token_map = TokenMap::default(); | 23 | syntax_node_to_token_tree(ast.syntax()) |
30 | let node = ast.syntax(); | ||
31 | let tt = convert_tt(&mut token_map, node.text_range().start(), node)?; | ||
32 | Some((tt, token_map)) | ||
33 | } | 24 | } |
34 | 25 | ||
35 | /// Convert the syntax node to a `TokenTree` (what macro | 26 | /// Convert the syntax node to a `TokenTree` (what macro |
36 | /// will consume). | 27 | /// will consume). |
37 | pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { | 28 | pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { |
38 | let mut token_map = TokenMap::default(); | 29 | let global_offset = node.text_range().start(); |
39 | let tt = convert_tt(&mut token_map, node.text_range().start(), node)?; | 30 | let mut c = Convertor { map: TokenMap::default(), global_offset, next_id: 0 }; |
40 | Some((tt, token_map)) | 31 | let subtree = c.go(node)?; |
32 | Some((subtree, c.map)) | ||
41 | } | 33 | } |
42 | 34 | ||
43 | // The following items are what `rustc` macro can be parsed into : | 35 | // The following items are what `rustc` macro can be parsed into : |
@@ -55,7 +47,7 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke | |||
55 | pub fn token_tree_to_syntax_node( | 47 | pub fn token_tree_to_syntax_node( |
56 | tt: &tt::Subtree, | 48 | tt: &tt::Subtree, |
57 | fragment_kind: FragmentKind, | 49 | fragment_kind: FragmentKind, |
58 | ) -> Result<(Parse<SyntaxNode>, RevTokenMap), ExpandError> { | 50 | ) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> { |
59 | let tmp; | 51 | let tmp; |
60 | let tokens = match tt { | 52 | let tokens = match tt { |
61 | tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(), | 53 | tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(), |
@@ -78,31 +70,17 @@ pub fn token_tree_to_syntax_node( | |||
78 | 70 | ||
79 | impl TokenMap { | 71 | impl TokenMap { |
80 | pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> { | 72 | pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> { |
81 | let (idx, _) = | 73 | let &(token_id, _) = self.entries.iter().find(|(_, range)| *range == relative_range)?; |
82 | self.tokens.iter().enumerate().find(|(_, range)| **range == relative_range)?; | 74 | Some(token_id) |
83 | Some(tt::TokenId(idx as u32)) | ||
84 | } | ||
85 | |||
86 | pub fn relative_range_of(&self, token_id: tt::TokenId) -> Option<TextRange> { | ||
87 | let idx = token_id.0 as usize; | ||
88 | self.tokens.get(idx).copied() | ||
89 | } | ||
90 | |||
91 | fn alloc(&mut self, relative_range: TextRange) -> tt::TokenId { | ||
92 | let id = self.tokens.len(); | ||
93 | self.tokens.push(relative_range); | ||
94 | tt::TokenId(id as u32) | ||
95 | } | 75 | } |
96 | } | ||
97 | 76 | ||
98 | impl RevTokenMap { | ||
99 | pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TextRange> { | 77 | pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TextRange> { |
100 | let &(r, _) = self.ranges.iter().find(|(_, tid)| *tid == token_id)?; | 78 | let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?; |
101 | Some(r) | 79 | Some(range) |
102 | } | 80 | } |
103 | 81 | ||
104 | fn add(&mut self, relative_range: TextRange, token_id: tt::TokenId) { | 82 | fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) { |
105 | self.ranges.push((relative_range, token_id.clone())) | 83 | self.entries.push((token_id, relative_range)); |
106 | } | 84 | } |
107 | } | 85 | } |
108 | 86 | ||
@@ -167,84 +145,98 @@ fn convert_doc_comment(token: &ra_syntax::SyntaxToken) -> Option<Vec<tt::TokenTr | |||
167 | } | 145 | } |
168 | } | 146 | } |
169 | 147 | ||
170 | fn convert_tt( | 148 | struct Convertor { |
171 | token_map: &mut TokenMap, | 149 | map: TokenMap, |
172 | global_offset: TextUnit, | 150 | global_offset: TextUnit, |
173 | tt: &SyntaxNode, | 151 | next_id: u32, |
174 | ) -> Option<tt::Subtree> { | 152 | } |
175 | // This tree is empty | ||
176 | if tt.first_child_or_token().is_none() { | ||
177 | return Some(tt::Subtree { token_trees: vec![], delimiter: tt::Delimiter::None }); | ||
178 | } | ||
179 | 153 | ||
180 | let first_child = tt.first_child_or_token()?; | 154 | impl Convertor { |
181 | let last_child = tt.last_child_or_token()?; | 155 | fn go(&mut self, tt: &SyntaxNode) -> Option<tt::Subtree> { |
182 | let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { | 156 | // This tree is empty |
183 | (T!['('], T![')']) => (tt::Delimiter::Parenthesis, true), | 157 | if tt.first_child_or_token().is_none() { |
184 | (T!['{'], T!['}']) => (tt::Delimiter::Brace, true), | 158 | return Some(tt::Subtree { token_trees: vec![], delimiter: tt::Delimiter::None }); |
185 | (T!['['], T![']']) => (tt::Delimiter::Bracket, true), | 159 | } |
186 | _ => (tt::Delimiter::None, false), | ||
187 | }; | ||
188 | 160 | ||
189 | let mut token_trees = Vec::new(); | 161 | let first_child = tt.first_child_or_token()?; |
190 | let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); | 162 | let last_child = tt.last_child_or_token()?; |
163 | let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { | ||
164 | (T!['('], T![')']) => (tt::Delimiter::Parenthesis, true), | ||
165 | (T!['{'], T!['}']) => (tt::Delimiter::Brace, true), | ||
166 | (T!['['], T![']']) => (tt::Delimiter::Bracket, true), | ||
167 | _ => (tt::Delimiter::None, false), | ||
168 | }; | ||
191 | 169 | ||
192 | while let Some(child) = child_iter.next() { | 170 | let mut token_trees = Vec::new(); |
193 | if skip_first && (child == first_child || child == last_child) { | 171 | let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); |
194 | continue; | ||
195 | } | ||
196 | 172 | ||
197 | match child { | 173 | while let Some(child) = child_iter.next() { |
198 | NodeOrToken::Token(token) => { | 174 | if skip_first && (child == first_child || child == last_child) { |
199 | if let Some(doc_tokens) = convert_doc_comment(&token) { | 175 | continue; |
200 | token_trees.extend(doc_tokens); | 176 | } |
201 | } else if token.kind().is_trivia() { | 177 | |
202 | continue; | 178 | match child { |
203 | } else if token.kind().is_punct() { | 179 | NodeOrToken::Token(token) => { |
204 | assert!(token.text().len() == 1, "Input ast::token punct must be single char."); | 180 | if let Some(doc_tokens) = convert_doc_comment(&token) { |
205 | let char = token.text().chars().next().unwrap(); | 181 | token_trees.extend(doc_tokens); |
206 | 182 | } else if token.kind().is_trivia() { | |
207 | let spacing = match child_iter.peek() { | 183 | continue; |
208 | Some(NodeOrToken::Token(token)) => { | 184 | } else if token.kind().is_punct() { |
209 | if token.kind().is_punct() { | 185 | assert!( |
210 | tt::Spacing::Joint | 186 | token.text().len() == 1, |
211 | } else { | 187 | "Input ast::token punct must be single char." |
212 | tt::Spacing::Alone | 188 | ); |
189 | let char = token.text().chars().next().unwrap(); | ||
190 | |||
191 | let spacing = match child_iter.peek() { | ||
192 | Some(NodeOrToken::Token(token)) => { | ||
193 | if token.kind().is_punct() { | ||
194 | tt::Spacing::Joint | ||
195 | } else { | ||
196 | tt::Spacing::Alone | ||
197 | } | ||
213 | } | 198 | } |
214 | } | 199 | _ => tt::Spacing::Alone, |
215 | _ => tt::Spacing::Alone, | ||
216 | }; | ||
217 | |||
218 | token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); | ||
219 | } else { | ||
220 | let child: tt::TokenTree = | ||
221 | if token.kind() == T![true] || token.kind() == T![false] { | ||
222 | tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() | ||
223 | } else if token.kind().is_keyword() | ||
224 | || token.kind() == IDENT | ||
225 | || token.kind() == LIFETIME | ||
226 | { | ||
227 | let relative_range = token.text_range() - global_offset; | ||
228 | let id = token_map.alloc(relative_range); | ||
229 | let text = token.text().clone(); | ||
230 | tt::Leaf::from(tt::Ident { text, id }).into() | ||
231 | } else if token.kind().is_literal() { | ||
232 | tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() | ||
233 | } else { | ||
234 | return None; | ||
235 | }; | 200 | }; |
201 | |||
202 | token_trees.push(tt::Leaf::from(tt::Punct { char, spacing }).into()); | ||
203 | } else { | ||
204 | let child: tt::TokenTree = | ||
205 | if token.kind() == T![true] || token.kind() == T![false] { | ||
206 | tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() | ||
207 | } else if token.kind().is_keyword() | ||
208 | || token.kind() == IDENT | ||
209 | || token.kind() == LIFETIME | ||
210 | { | ||
211 | let id = self.alloc(token.text_range()); | ||
212 | let text = token.text().clone(); | ||
213 | tt::Leaf::from(tt::Ident { text, id }).into() | ||
214 | } else if token.kind().is_literal() { | ||
215 | tt::Leaf::from(tt::Literal { text: token.text().clone() }).into() | ||
216 | } else { | ||
217 | return None; | ||
218 | }; | ||
219 | token_trees.push(child); | ||
220 | } | ||
221 | } | ||
222 | NodeOrToken::Node(node) => { | ||
223 | let child = self.go(&node)?.into(); | ||
236 | token_trees.push(child); | 224 | token_trees.push(child); |
237 | } | 225 | } |
238 | } | 226 | }; |
239 | NodeOrToken::Node(node) => { | 227 | } |
240 | let child = convert_tt(token_map, global_offset, &node)?.into(); | 228 | |
241 | token_trees.push(child); | 229 | let res = tt::Subtree { delimiter, token_trees }; |
242 | } | 230 | Some(res) |
243 | }; | ||
244 | } | 231 | } |
245 | 232 | ||
246 | let res = tt::Subtree { delimiter, token_trees }; | 233 | fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId { |
247 | Some(res) | 234 | let relative_range = absolute_range - self.global_offset; |
235 | let token_id = tt::TokenId(self.next_id); | ||
236 | self.next_id += 1; | ||
237 | self.map.insert(token_id, relative_range); | ||
238 | token_id | ||
239 | } | ||
248 | } | 240 | } |
249 | 241 | ||
250 | struct TtTreeSink<'a> { | 242 | struct TtTreeSink<'a> { |
@@ -252,7 +244,7 @@ struct TtTreeSink<'a> { | |||
252 | cursor: Cursor<'a>, | 244 | cursor: Cursor<'a>, |
253 | text_pos: TextUnit, | 245 | text_pos: TextUnit, |
254 | inner: SyntaxTreeBuilder, | 246 | inner: SyntaxTreeBuilder, |
255 | range_map: RevTokenMap, | 247 | token_map: TokenMap, |
256 | 248 | ||
257 | // Number of roots | 249 | // Number of roots |
258 | // Use for detect ill-form tree which is not single root | 250 | // Use for detect ill-form tree which is not single root |
@@ -267,12 +259,12 @@ impl<'a> TtTreeSink<'a> { | |||
267 | text_pos: 0.into(), | 259 | text_pos: 0.into(), |
268 | inner: SyntaxTreeBuilder::default(), | 260 | inner: SyntaxTreeBuilder::default(), |
269 | roots: smallvec::SmallVec::new(), | 261 | roots: smallvec::SmallVec::new(), |
270 | range_map: RevTokenMap::default(), | 262 | token_map: TokenMap::default(), |
271 | } | 263 | } |
272 | } | 264 | } |
273 | 265 | ||
274 | fn finish(self) -> (Parse<SyntaxNode>, RevTokenMap) { | 266 | fn finish(self) -> (Parse<SyntaxNode>, TokenMap) { |
275 | (self.inner.finish(), self.range_map) | 267 | (self.inner.finish(), self.token_map) |
276 | } | 268 | } |
277 | } | 269 | } |
278 | 270 | ||
@@ -308,7 +300,7 @@ impl<'a> TreeSink for TtTreeSink<'a> { | |||
308 | if kind == IDENT { | 300 | if kind == IDENT { |
309 | let range = | 301 | let range = |
310 | TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); | 302 | TextRange::offset_len(self.text_pos, TextUnit::of_str(&ident.text)); |
311 | self.range_map.add(range, ident.id); | 303 | self.token_map.insert(ident.id, range); |
312 | } | 304 | } |
313 | } | 305 | } |
314 | 306 | ||