diff options
Diffstat (limited to 'crates/ra_mbe/src')
-rw-r--r-- | crates/ra_mbe/src/lib.rs | 57 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_parser.rs | 3 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_source.rs | 354 | ||||
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 47 |
4 files changed, 243 insertions, 218 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index a21ea4dbc..4126854d1 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs | |||
@@ -39,7 +39,7 @@ pub enum ExpandError { | |||
39 | BindingError(String), | 39 | BindingError(String), |
40 | } | 40 | } |
41 | 41 | ||
42 | pub use crate::syntax_bridge::{ast_to_token_tree, token_tree_to_ast_item_list}; | 42 | pub use crate::syntax_bridge::{ast_to_token_tree, token_tree_to_ast_item_list, syntax_node_to_token_tree}; |
43 | 43 | ||
44 | /// This struct contains AST for a single `macro_rules` definition. What might | 44 | /// This struct contains AST for a single `macro_rules` definition. What might |
45 | /// be very confusing is that AST has almost exactly the same shape as | 45 | /// be very confusing is that AST has almost exactly the same shape as |
@@ -192,6 +192,15 @@ impl_froms!(TokenTree: Leaf, Subtree); | |||
192 | pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { | 192 | pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) { |
193 | let expanded = expand(rules, invocation); | 193 | let expanded = expand(rules, invocation); |
194 | assert_eq!(expanded.to_string(), expansion); | 194 | assert_eq!(expanded.to_string(), expansion); |
195 | |||
196 | let tree = token_tree_to_ast_item_list(&expanded); | ||
197 | |||
198 | // Eat all white space by parse it back and forth | ||
199 | let expansion = ast::SourceFile::parse(expansion); | ||
200 | let expansion = syntax_node_to_token_tree(expansion.syntax()).unwrap().0; | ||
201 | let file = token_tree_to_ast_item_list(&expansion); | ||
202 | |||
203 | assert_eq!(tree.syntax().debug_dump().trim(), file.syntax().debug_dump().trim()); | ||
195 | } | 204 | } |
196 | 205 | ||
197 | #[test] | 206 | #[test] |
@@ -288,6 +297,36 @@ impl_froms!(TokenTree: Leaf, Subtree); | |||
288 | } | 297 | } |
289 | 298 | ||
290 | #[test] | 299 | #[test] |
300 | fn test_match_group_pattern_with_multiple_defs() { | ||
301 | let rules = create_rules( | ||
302 | r#" | ||
303 | macro_rules! foo { | ||
304 | ($ ($ i:ident),*) => ( struct Bar { $ ( | ||
305 | fn $ i {} | ||
306 | )*} ); | ||
307 | } | ||
308 | "#, | ||
309 | ); | ||
310 | |||
311 | assert_expansion(&rules, "foo! { foo, bar }", "struct Bar {fn foo {} fn bar {}}"); | ||
312 | } | ||
313 | |||
314 | #[test] | ||
315 | fn test_match_group_pattern_with_multiple_statement() { | ||
316 | let rules = create_rules( | ||
317 | r#" | ||
318 | macro_rules! foo { | ||
319 | ($ ($ i:ident),*) => ( fn baz { $ ( | ||
320 | $ i (); | ||
321 | )*} ); | ||
322 | } | ||
323 | "#, | ||
324 | ); | ||
325 | |||
326 | assert_expansion(&rules, "foo! { foo, bar }", "fn baz {foo () ; bar () ;}"); | ||
327 | } | ||
328 | |||
329 | #[test] | ||
291 | fn expand_to_item_list() { | 330 | fn expand_to_item_list() { |
292 | let rules = create_rules( | 331 | let rules = create_rules( |
293 | " | 332 | " |
@@ -415,7 +454,7 @@ SOURCE_FILE@[0; 40) | |||
415 | assert_expansion( | 454 | assert_expansion( |
416 | &rules, | 455 | &rules, |
417 | "foo! { bar::<u8>::baz::<u8> }", | 456 | "foo! { bar::<u8>::baz::<u8> }", |
418 | "fn foo () {let a = bar ::< u8 > ::baz ::< u8 > ;}", | 457 | "fn foo () {let a = bar :: < u8 > :: baz :: < u8 > ;}", |
419 | ); | 458 | ); |
420 | } | 459 | } |
421 | 460 | ||
@@ -432,4 +471,18 @@ SOURCE_FILE@[0; 40) | |||
432 | ); | 471 | ); |
433 | assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); | 472 | assert_expansion(&rules, "foo! { foo, bar }", "fn foo () {let a = foo ; let b = bar ;}"); |
434 | } | 473 | } |
474 | |||
475 | #[test] | ||
476 | fn test_path_with_path() { | ||
477 | let rules = create_rules( | ||
478 | r#" | ||
479 | macro_rules! foo { | ||
480 | ($ i:path) => { | ||
481 | fn foo() { let a = $ i :: bar; } | ||
482 | } | ||
483 | } | ||
484 | "#, | ||
485 | ); | ||
486 | assert_expansion(&rules, "foo! { foo }", "fn foo () {let a = foo :: bar ;}"); | ||
487 | } | ||
435 | } | 488 | } |
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs index ce39a40bb..164240d92 100644 --- a/crates/ra_mbe/src/subtree_parser.rs +++ b/crates/ra_mbe/src/subtree_parser.rs | |||
@@ -34,8 +34,7 @@ impl<'a> Parser<'a> { | |||
34 | where | 34 | where |
35 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), | 35 | F: FnOnce(&dyn TokenSource, &mut dyn TreeSink), |
36 | { | 36 | { |
37 | let mut src = SubtreeTokenSource::new(self.subtree); | 37 | let mut src = SubtreeTokenSource::new(&self.subtree.token_trees[*self.cur_pos..]); |
38 | src.start_from_nth(*self.cur_pos); | ||
39 | let mut sink = OffsetTokenSink { token_pos: 0 }; | 38 | let mut sink = OffsetTokenSink { token_pos: 0 }; |
40 | 39 | ||
41 | f(&src, &mut sink); | 40 | f(&src, &mut sink); |
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index 4b37c2bda..6aa20057e 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs | |||
@@ -2,6 +2,64 @@ use ra_parser::{TokenSource}; | |||
2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; | 2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; |
3 | use std::cell::{RefCell}; | 3 | use std::cell::{RefCell}; |
4 | 4 | ||
5 | // A Sequece of Token, | ||
6 | #[derive(Debug, Clone, Eq, PartialEq)] | ||
7 | pub(super) enum TokenSeq<'a> { | ||
8 | Subtree(&'a tt::Subtree), | ||
9 | Seq(&'a [tt::TokenTree]), | ||
10 | } | ||
11 | |||
12 | impl<'a> From<&'a tt::Subtree> for TokenSeq<'a> { | ||
13 | fn from(s: &'a tt::Subtree) -> TokenSeq<'a> { | ||
14 | TokenSeq::Subtree(s) | ||
15 | } | ||
16 | } | ||
17 | |||
18 | impl<'a> From<&'a [tt::TokenTree]> for TokenSeq<'a> { | ||
19 | fn from(s: &'a [tt::TokenTree]) -> TokenSeq<'a> { | ||
20 | TokenSeq::Seq(s) | ||
21 | } | ||
22 | } | ||
23 | |||
24 | enum DelimToken<'a> { | ||
25 | Delim(&'a tt::Delimiter, bool), | ||
26 | Token(&'a tt::TokenTree), | ||
27 | End, | ||
28 | } | ||
29 | |||
30 | impl<'a> TokenSeq<'a> { | ||
31 | fn get(&self, pos: usize) -> DelimToken<'a> { | ||
32 | match self { | ||
33 | TokenSeq::Subtree(subtree) => { | ||
34 | let len = subtree.token_trees.len() + 2; | ||
35 | match pos { | ||
36 | p if p >= len => DelimToken::End, | ||
37 | p if p == len - 1 => DelimToken::Delim(&subtree.delimiter, true), | ||
38 | 0 => DelimToken::Delim(&subtree.delimiter, false), | ||
39 | p => DelimToken::Token(&subtree.token_trees[p - 1]), | ||
40 | } | ||
41 | } | ||
42 | TokenSeq::Seq(tokens) => { | ||
43 | tokens.get(pos).map(DelimToken::Token).unwrap_or(DelimToken::End) | ||
44 | } | ||
45 | } | ||
46 | } | ||
47 | |||
48 | fn len(&self) -> usize { | ||
49 | match self { | ||
50 | TokenSeq::Subtree(subtree) => subtree.token_trees.len() + 2, | ||
51 | TokenSeq::Seq(tokens) => tokens.len(), | ||
52 | } | ||
53 | } | ||
54 | |||
55 | fn child_slice(&self) -> &[tt::TokenTree] { | ||
56 | match self { | ||
57 | TokenSeq::Subtree(subtree) => &subtree.token_trees, | ||
58 | TokenSeq::Seq(tokens) => &tokens, | ||
59 | } | ||
60 | } | ||
61 | } | ||
62 | |||
5 | #[derive(Debug, Clone, Eq, PartialEq)] | 63 | #[derive(Debug, Clone, Eq, PartialEq)] |
6 | struct TtToken { | 64 | struct TtToken { |
7 | pub kind: SyntaxKind, | 65 | pub kind: SyntaxKind, |
@@ -12,29 +70,27 @@ struct TtToken { | |||
12 | 70 | ||
13 | #[derive(Debug, Clone, Eq, PartialEq)] | 71 | #[derive(Debug, Clone, Eq, PartialEq)] |
14 | enum WalkCursor { | 72 | enum WalkCursor { |
15 | DelimiterBegin(Option<TtToken>), | 73 | Token(usize, TtToken), |
16 | Token(usize, Option<TtToken>), | ||
17 | DelimiterEnd(Option<TtToken>), | ||
18 | Eof, | 74 | Eof, |
19 | } | 75 | } |
20 | 76 | ||
21 | #[derive(Debug)] | 77 | #[derive(Debug)] |
22 | struct SubTreeWalker<'a> { | 78 | struct SubTreeWalker<'a> { |
23 | pos: usize, | 79 | pos: usize, |
24 | stack: Vec<(&'a tt::Subtree, Option<usize>)>, | 80 | stack: Vec<(TokenSeq<'a>, usize)>, |
25 | cursor: WalkCursor, | 81 | cursor: WalkCursor, |
26 | last_steps: Vec<usize>, | 82 | last_steps: Vec<usize>, |
27 | subtree: &'a tt::Subtree, | 83 | ts: TokenSeq<'a>, |
28 | } | 84 | } |
29 | 85 | ||
30 | impl<'a> SubTreeWalker<'a> { | 86 | impl<'a> SubTreeWalker<'a> { |
31 | fn new(subtree: &tt::Subtree) -> SubTreeWalker { | 87 | fn new(ts: TokenSeq<'a>) -> SubTreeWalker { |
32 | let mut res = SubTreeWalker { | 88 | let mut res = SubTreeWalker { |
33 | pos: 0, | 89 | pos: 0, |
34 | stack: vec![], | 90 | stack: vec![], |
35 | cursor: WalkCursor::Eof, | 91 | cursor: WalkCursor::Eof, |
36 | last_steps: vec![], | 92 | last_steps: vec![], |
37 | subtree, | 93 | ts, |
38 | }; | 94 | }; |
39 | 95 | ||
40 | res.reset(); | 96 | res.reset(); |
@@ -47,209 +103,106 @@ impl<'a> SubTreeWalker<'a> { | |||
47 | 103 | ||
48 | fn reset(&mut self) { | 104 | fn reset(&mut self) { |
49 | self.pos = 0; | 105 | self.pos = 0; |
50 | self.stack = vec![(self.subtree, None)]; | 106 | self.stack = vec![]; |
51 | self.cursor = WalkCursor::DelimiterBegin(convert_delim(self.subtree.delimiter, false)); | ||
52 | self.last_steps = vec![]; | 107 | self.last_steps = vec![]; |
53 | 108 | ||
54 | while self.is_empty_delimiter() { | 109 | self.cursor = match self.ts.get(0) { |
55 | self.forward_unchecked(); | 110 | DelimToken::Token(token) => match token { |
56 | } | 111 | tt::TokenTree::Subtree(subtree) => { |
57 | } | 112 | WalkCursor::Token(0, convert_delim(subtree.delimiter, false)) |
58 | 113 | } | |
59 | // This funciton will fast forward the cursor, | 114 | tt::TokenTree::Leaf(leaf) => { |
60 | // Such that backward will stop at `start_pos` point | 115 | let next_tokens = self.ts.child_slice(); |
61 | fn start_from_nth(&mut self, start_pos: usize) { | 116 | WalkCursor::Token(0, convert_leaf(&next_tokens, leaf)) |
62 | self.reset(); | 117 | } |
63 | self.pos = start_pos; | 118 | }, |
64 | self.cursor = self.walk_token(start_pos, 0, false); | 119 | DelimToken::Delim(delim, is_end) => { |
65 | 120 | assert!(!is_end); | |
66 | while self.is_empty_delimiter() { | 121 | WalkCursor::Token(0, convert_delim(*delim, false)) |
67 | self.forward_unchecked(); | 122 | } |
123 | DelimToken::End => WalkCursor::Eof, | ||
68 | } | 124 | } |
69 | } | 125 | } |
70 | 126 | ||
71 | fn current(&self) -> Option<&TtToken> { | 127 | fn current(&self) -> Option<&TtToken> { |
72 | match &self.cursor { | 128 | match &self.cursor { |
73 | WalkCursor::DelimiterBegin(t) => t.as_ref(), | 129 | WalkCursor::Token(_, t) => Some(t), |
74 | WalkCursor::Token(_, t) => t.as_ref(), | ||
75 | WalkCursor::DelimiterEnd(t) => t.as_ref(), | ||
76 | WalkCursor::Eof => None, | 130 | WalkCursor::Eof => None, |
77 | } | 131 | } |
78 | } | 132 | } |
79 | 133 | ||
80 | fn is_empty_delimiter(&self) -> bool { | 134 | fn top(&self) -> &TokenSeq { |
81 | match &self.cursor { | 135 | self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts) |
82 | WalkCursor::DelimiterBegin(None) => true, | ||
83 | WalkCursor::DelimiterEnd(None) => true, | ||
84 | _ => false, | ||
85 | } | ||
86 | } | 136 | } |
87 | 137 | ||
88 | /// Move cursor backward by 1 step with empty checking | 138 | /// Move cursor backward by 1 step |
89 | fn backward(&mut self) { | 139 | fn backward(&mut self) { |
90 | if self.last_steps.is_empty() { | 140 | if self.last_steps.is_empty() { |
91 | return; | 141 | return; |
92 | } | 142 | } |
143 | |||
93 | self.pos -= 1; | 144 | self.pos -= 1; |
94 | loop { | 145 | let last_step = self.last_steps.pop().unwrap(); |
95 | self.backward_unchecked(); | ||
96 | // Skip Empty delimiter | ||
97 | if self.last_steps.is_empty() || !self.is_empty_delimiter() { | ||
98 | break; | ||
99 | } | ||
100 | } | ||
101 | 146 | ||
102 | // Move forward if it is empty delimiter | 147 | self.cursor = match self.cursor { |
103 | if self.last_steps.is_empty() { | 148 | WalkCursor::Token(idx, _) => self.walk_token(idx, last_step, true), |
104 | while self.is_empty_delimiter() { | 149 | WalkCursor::Eof => { |
105 | self.forward_unchecked(); | 150 | let len = self.top().len(); |
151 | self.walk_token(len, last_step, true) | ||
106 | } | 152 | } |
107 | } | 153 | } |
108 | } | 154 | } |
109 | 155 | ||
110 | /// Move cursor backward by 1 step without empty check | 156 | /// Move cursor forward by 1 step |
111 | /// | ||
112 | /// Depends on the current state of cursor: | ||
113 | /// | ||
114 | /// * Delimiter Begin => Pop the stack, goto last walking token (`walk_token`) | ||
115 | /// * Token => Goto prev token (`walk_token`) | ||
116 | /// * Delimiter End => Goto the last child token (`walk_token`) | ||
117 | /// * Eof => push the root subtree, and set it as Delimiter End | ||
118 | fn backward_unchecked(&mut self) { | ||
119 | if self.last_steps.is_empty() { | ||
120 | return; | ||
121 | } | ||
122 | |||
123 | let last_step = self.last_steps.pop().unwrap(); | ||
124 | let do_walk_token = match self.cursor { | ||
125 | WalkCursor::DelimiterBegin(_) => None, | ||
126 | WalkCursor::Token(u, _) => Some(u), | ||
127 | WalkCursor::DelimiterEnd(_) => { | ||
128 | let (top, _) = self.stack.last().unwrap(); | ||
129 | Some(top.token_trees.len()) | ||
130 | } | ||
131 | WalkCursor::Eof => None, | ||
132 | }; | ||
133 | |||
134 | self.cursor = match do_walk_token { | ||
135 | Some(u) => self.walk_token(u, last_step, true), | ||
136 | None => match self.cursor { | ||
137 | WalkCursor::Eof => { | ||
138 | self.stack.push((self.subtree, None)); | ||
139 | WalkCursor::DelimiterEnd(convert_delim( | ||
140 | self.stack.last().unwrap().0.delimiter, | ||
141 | true, | ||
142 | )) | ||
143 | } | ||
144 | _ => { | ||
145 | let (_, last_top_cursor) = self.stack.pop().unwrap(); | ||
146 | assert!(!self.stack.is_empty()); | ||
147 | |||
148 | self.walk_token(last_top_cursor.unwrap(), last_step, true) | ||
149 | } | ||
150 | }, | ||
151 | }; | ||
152 | } | ||
153 | |||
154 | /// Move cursor forward by 1 step with empty checking | ||
155 | fn forward(&mut self) { | 157 | fn forward(&mut self) { |
156 | if self.is_eof() { | 158 | if self.is_eof() { |
157 | return; | 159 | return; |
158 | } | 160 | } |
159 | |||
160 | self.pos += 1; | 161 | self.pos += 1; |
161 | loop { | ||
162 | self.forward_unchecked(); | ||
163 | if !self.is_empty_delimiter() { | ||
164 | break; | ||
165 | } | ||
166 | } | ||
167 | } | ||
168 | |||
169 | /// Move cursor forward by 1 step without empty checking | ||
170 | /// | ||
171 | /// Depends on the current state of cursor: | ||
172 | /// | ||
173 | /// * Delimiter Begin => Goto the first child token (`walk_token`) | ||
174 | /// * Token => Goto next token (`walk_token`) | ||
175 | /// * Delimiter End => Pop the stack, goto last walking token (`walk_token`) | ||
176 | /// | ||
177 | fn forward_unchecked(&mut self) { | ||
178 | if self.is_eof() { | ||
179 | return; | ||
180 | } | ||
181 | 162 | ||
182 | let step = self.current().map(|x| x.n_tokens).unwrap_or(1); | 163 | let step = self.current().map(|x| x.n_tokens).unwrap_or(1); |
183 | self.last_steps.push(step); | 164 | self.last_steps.push(step); |
184 | 165 | ||
185 | let do_walk_token = match self.cursor { | 166 | if let WalkCursor::Token(u, _) = self.cursor { |
186 | WalkCursor::DelimiterBegin(_) => Some((0, 0)), | 167 | self.cursor = self.walk_token(u, step, false) |
187 | WalkCursor::Token(u, _) => Some((u, step)), | 168 | } |
188 | WalkCursor::DelimiterEnd(_) => None, | ||
189 | _ => unreachable!(), | ||
190 | }; | ||
191 | |||
192 | self.cursor = match do_walk_token { | ||
193 | Some((u, step)) => self.walk_token(u, step, false), | ||
194 | None => { | ||
195 | let (_, last_top_idx) = self.stack.pop().unwrap(); | ||
196 | match self.stack.last() { | ||
197 | Some(_) => self.walk_token(last_top_idx.unwrap(), 1, false), | ||
198 | None => WalkCursor::Eof, | ||
199 | } | ||
200 | } | ||
201 | }; | ||
202 | } | 169 | } |
203 | 170 | ||
204 | /// Traversal child token | 171 | /// Traversal child token |
205 | /// Depends on the new position, it returns: | ||
206 | /// | ||
207 | /// * new position < 0 => DelimiterBegin | ||
208 | /// * new position > token_tree.len() => DelimiterEnd | ||
209 | /// * if new position is a subtree, depends on traversal direction: | ||
210 | /// ** backward => DelimiterEnd | ||
211 | /// ** forward => DelimiterBegin | ||
212 | /// * if new psoition is a leaf, return walk_leaf() | ||
213 | fn walk_token(&mut self, pos: usize, offset: usize, backward: bool) -> WalkCursor { | 172 | fn walk_token(&mut self, pos: usize, offset: usize, backward: bool) -> WalkCursor { |
214 | let (top, _) = self.stack.last().unwrap(); | 173 | let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts); |
215 | 174 | ||
216 | if backward && pos < offset { | 175 | if backward && pos < offset { |
217 | return WalkCursor::DelimiterBegin(convert_delim( | 176 | let (_, last_idx) = self.stack.pop().unwrap(); |
218 | self.stack.last().unwrap().0.delimiter, | 177 | return self.walk_token(last_idx, offset, backward); |
219 | false, | ||
220 | )); | ||
221 | } | ||
222 | |||
223 | if !backward && pos + offset >= top.token_trees.len() { | ||
224 | return WalkCursor::DelimiterEnd(convert_delim( | ||
225 | self.stack.last().unwrap().0.delimiter, | ||
226 | true, | ||
227 | )); | ||
228 | } | 178 | } |
229 | 179 | ||
230 | let pos = if backward { pos - offset } else { pos + offset }; | 180 | let pos = if backward { pos - offset } else { pos + offset }; |
231 | 181 | ||
232 | match &top.token_trees[pos] { | 182 | match top.get(pos) { |
233 | tt::TokenTree::Subtree(subtree) => { | 183 | DelimToken::Token(token) => match token { |
234 | self.stack.push((subtree, Some(pos))); | 184 | tt::TokenTree::Subtree(subtree) => { |
235 | let delim = convert_delim(self.stack.last().unwrap().0.delimiter, backward); | 185 | let ts = TokenSeq::from(subtree); |
236 | if backward { | 186 | let new_idx = if backward { ts.len() - 1 } else { 0 }; |
237 | WalkCursor::DelimiterEnd(delim) | 187 | self.stack.push((ts, pos)); |
238 | } else { | 188 | WalkCursor::Token(new_idx, convert_delim(subtree.delimiter, backward)) |
239 | WalkCursor::DelimiterBegin(delim) | 189 | } |
190 | tt::TokenTree::Leaf(leaf) => { | ||
191 | let next_tokens = top.child_slice(); | ||
192 | WalkCursor::Token(pos, convert_leaf(&next_tokens[pos..], leaf)) | ||
240 | } | 193 | } |
194 | }, | ||
195 | DelimToken::Delim(delim, is_end) => { | ||
196 | WalkCursor::Token(pos, convert_delim(*delim, is_end)) | ||
241 | } | 197 | } |
242 | tt::TokenTree::Leaf(leaf) => WalkCursor::Token(pos, Some(self.walk_leaf(leaf, pos))), | 198 | DelimToken::End => { |
243 | } | 199 | // it is the top level |
244 | } | 200 | if let Some((_, last_idx)) = self.stack.pop() { |
245 | 201 | assert!(!backward); | |
246 | fn walk_leaf(&mut self, leaf: &tt::Leaf, pos: usize) -> TtToken { | 202 | self.walk_token(last_idx, offset, backward) |
247 | match leaf { | 203 | } else { |
248 | tt::Leaf::Literal(l) => convert_literal(l), | 204 | WalkCursor::Eof |
249 | tt::Leaf::Ident(ident) => convert_ident(ident), | 205 | } |
250 | tt::Leaf::Punct(punct) => { | ||
251 | let (top, _) = self.stack.last().unwrap(); | ||
252 | convert_punct(punct, top, pos) | ||
253 | } | 206 | } |
254 | } | 207 | } |
255 | } | 208 | } |
@@ -263,27 +216,20 @@ pub(crate) trait Querier { | |||
263 | #[derive(Debug)] | 216 | #[derive(Debug)] |
264 | pub(crate) struct WalkerOwner<'a> { | 217 | pub(crate) struct WalkerOwner<'a> { |
265 | walker: RefCell<SubTreeWalker<'a>>, | 218 | walker: RefCell<SubTreeWalker<'a>>, |
266 | offset: usize, | ||
267 | } | 219 | } |
268 | 220 | ||
269 | impl<'a> WalkerOwner<'a> { | 221 | impl<'a> WalkerOwner<'a> { |
270 | fn new(subtree: &'a tt::Subtree) -> Self { | 222 | fn new<I: Into<TokenSeq<'a>>>(ts: I) -> Self { |
271 | WalkerOwner { walker: RefCell::new(SubTreeWalker::new(subtree)), offset: 0 } | 223 | WalkerOwner { walker: RefCell::new(SubTreeWalker::new(ts.into())) } |
272 | } | 224 | } |
273 | 225 | ||
274 | fn get<'b>(&self, pos: usize) -> Option<TtToken> { | 226 | fn get<'b>(&self, pos: usize) -> Option<TtToken> { |
275 | self.set_walker_pos(pos); | 227 | self.set_pos(pos); |
276 | let walker = self.walker.borrow(); | 228 | let walker = self.walker.borrow(); |
277 | walker.current().cloned() | 229 | walker.current().cloned() |
278 | } | 230 | } |
279 | 231 | ||
280 | fn start_from_nth(&mut self, pos: usize) { | 232 | fn set_pos(&self, pos: usize) { |
281 | self.offset = pos; | ||
282 | self.walker.borrow_mut().start_from_nth(pos); | ||
283 | } | ||
284 | |||
285 | fn set_walker_pos(&self, mut pos: usize) { | ||
286 | pos += self.offset; | ||
287 | let mut walker = self.walker.borrow_mut(); | 233 | let mut walker = self.walker.borrow_mut(); |
288 | while pos > walker.pos && !walker.is_eof() { | 234 | while pos > walker.pos && !walker.is_eof() { |
289 | walker.forward(); | 235 | walker.forward(); |
@@ -294,19 +240,26 @@ impl<'a> WalkerOwner<'a> { | |||
294 | } | 240 | } |
295 | 241 | ||
296 | fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { | 242 | fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { |
297 | self.start_from_nth(self.offset); | ||
298 | |||
299 | let mut res = vec![]; | 243 | let mut res = vec![]; |
300 | let mut walker = self.walker.borrow_mut(); | 244 | let mut walker = self.walker.borrow_mut(); |
245 | walker.reset(); | ||
301 | 246 | ||
302 | while walker.pos - self.offset < n { | 247 | while walker.pos < n { |
303 | if let WalkCursor::Token(u, tt) = &walker.cursor { | 248 | if let WalkCursor::Token(u, tt) = &walker.cursor { |
304 | if walker.stack.len() == 1 { | 249 | // We only collect the topmost child |
305 | // We only collect the topmost child | 250 | if walker.stack.len() == 0 { |
306 | res.push(&walker.stack[0].0.token_trees[*u]); | 251 | for i in 0..tt.n_tokens { |
307 | if let Some(tt) = tt { | 252 | if let DelimToken::Token(token) = walker.ts.get(u + i) { |
308 | for i in 0..tt.n_tokens - 1 { | 253 | res.push(token); |
309 | res.push(&walker.stack[0].0.token_trees[u + i]); | 254 | } |
255 | } | ||
256 | } else if walker.stack.len() == 1 { | ||
257 | if let DelimToken::Delim(_, is_end) = walker.ts.get(*u) { | ||
258 | if !is_end { | ||
259 | let (_, last_idx) = &walker.stack[0]; | ||
260 | if let DelimToken::Token(token) = walker.ts.get(*last_idx) { | ||
261 | res.push(token); | ||
262 | } | ||
310 | } | 263 | } |
311 | } | 264 | } |
312 | } | 265 | } |
@@ -331,12 +284,8 @@ pub(crate) struct SubtreeTokenSource<'a> { | |||
331 | } | 284 | } |
332 | 285 | ||
333 | impl<'a> SubtreeTokenSource<'a> { | 286 | impl<'a> SubtreeTokenSource<'a> { |
334 | pub fn new(subtree: &tt::Subtree) -> SubtreeTokenSource { | 287 | pub fn new<I: Into<TokenSeq<'a>>>(ts: I) -> SubtreeTokenSource<'a> { |
335 | SubtreeTokenSource { walker: WalkerOwner::new(subtree) } | 288 | SubtreeTokenSource { walker: WalkerOwner::new(ts) } |
336 | } | ||
337 | |||
338 | pub fn start_from_nth(&mut self, n: usize) { | ||
339 | self.walker.start_from_nth(n); | ||
340 | } | 289 | } |
341 | 290 | ||
342 | pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> | 291 | pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> |
@@ -467,18 +416,18 @@ where | |||
467 | None | 416 | None |
468 | } | 417 | } |
469 | 418 | ||
470 | fn convert_delim(d: tt::Delimiter, closing: bool) -> Option<TtToken> { | 419 | fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken { |
471 | let (kinds, texts) = match d { | 420 | let (kinds, texts) = match d { |
472 | tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), | 421 | tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), |
473 | tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), | 422 | tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), |
474 | tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), | 423 | tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), |
475 | tt::Delimiter::None => return None, | 424 | tt::Delimiter::None => ([L_DOLLAR, R_DOLLAR], ""), |
476 | }; | 425 | }; |
477 | 426 | ||
478 | let idx = closing as usize; | 427 | let idx = closing as usize; |
479 | let kind = kinds[idx]; | 428 | let kind = kinds[idx]; |
480 | let text = &texts[idx..texts.len() - (1 - idx)]; | 429 | let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" }; |
481 | Some(TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 }) | 430 | TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 } |
482 | } | 431 | } |
483 | 432 | ||
484 | fn convert_literal(l: &tt::Literal) -> TtToken { | 433 | fn convert_literal(l: &tt::Literal) -> TtToken { |
@@ -495,8 +444,9 @@ fn convert_ident(ident: &tt::Ident) -> TtToken { | |||
495 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } | 444 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } |
496 | } | 445 | } |
497 | 446 | ||
498 | fn convert_punct(p: &tt::Punct, parent: &tt::Subtree, next: usize) -> TtToken { | 447 | fn convert_punct(p: &tt::Punct, next_tokens: &[tt::TokenTree]) -> TtToken { |
499 | let iter = parent.token_trees[next + 1..].iter(); | 448 | let mut iter = next_tokens.iter(); |
449 | iter.next(); | ||
500 | let mut peek = TokenPeek::new(iter); | 450 | let mut peek = TokenPeek::new(iter); |
501 | 451 | ||
502 | if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { | 452 | if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { |
@@ -519,3 +469,11 @@ fn convert_punct(p: &tt::Punct, parent: &tt::Subtree, next: usize) -> TtToken { | |||
519 | TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 } | 469 | TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 } |
520 | } | 470 | } |
521 | } | 471 | } |
472 | |||
473 | fn convert_leaf(tokens: &[tt::TokenTree], leaf: &tt::Leaf) -> TtToken { | ||
474 | match leaf { | ||
475 | tt::Leaf::Literal(l) => convert_literal(l), | ||
476 | tt::Leaf::Ident(ident) => convert_ident(ident), | ||
477 | tt::Leaf::Punct(punct) => convert_punct(punct, tokens), | ||
478 | } | ||
479 | } | ||
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index 19c17bd55..28ded7870 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -22,6 +22,14 @@ pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap) | |||
22 | Some((tt, token_map)) | 22 | Some((tt, token_map)) |
23 | } | 23 | } |
24 | 24 | ||
25 | /// Convert the syntax node to a `TokenTree` (what macro | ||
26 | /// will consume). | ||
27 | pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { | ||
28 | let mut token_map = TokenMap::default(); | ||
29 | let tt = convert_tt(&mut token_map, node.range().start(), node)?; | ||
30 | Some((tt, token_map)) | ||
31 | } | ||
32 | |||
25 | /// Parses the token tree (result of macro expansion) as a sequence of items | 33 | /// Parses the token tree (result of macro expansion) as a sequence of items |
26 | pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { | 34 | pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> { |
27 | let token_source = SubtreeTokenSource::new(tt); | 35 | let token_source = SubtreeTokenSource::new(tt); |
@@ -51,15 +59,17 @@ fn convert_tt( | |||
51 | ) -> Option<tt::Subtree> { | 59 | ) -> Option<tt::Subtree> { |
52 | let first_child = tt.first_child_or_token()?; | 60 | let first_child = tt.first_child_or_token()?; |
53 | let last_child = tt.last_child_or_token()?; | 61 | let last_child = tt.last_child_or_token()?; |
54 | let delimiter = match (first_child.kind(), last_child.kind()) { | 62 | let (delimiter, skip_first) = match (first_child.kind(), last_child.kind()) { |
55 | (L_PAREN, R_PAREN) => tt::Delimiter::Parenthesis, | 63 | (L_PAREN, R_PAREN) => (tt::Delimiter::Parenthesis, true), |
56 | (L_CURLY, R_CURLY) => tt::Delimiter::Brace, | 64 | (L_CURLY, R_CURLY) => (tt::Delimiter::Brace, true), |
57 | (L_BRACK, R_BRACK) => tt::Delimiter::Bracket, | 65 | (L_BRACK, R_BRACK) => (tt::Delimiter::Bracket, true), |
58 | _ => return None, | 66 | _ => (tt::Delimiter::None, false), |
59 | }; | 67 | }; |
68 | |||
60 | let mut token_trees = Vec::new(); | 69 | let mut token_trees = Vec::new(); |
61 | for child in tt.children_with_tokens().skip(1) { | 70 | for child in tt.children_with_tokens().skip(skip_first as usize) { |
62 | if child == first_child || child == last_child || child.kind().is_trivia() { | 71 | if (skip_first && (child == first_child || child == last_child)) || child.kind().is_trivia() |
72 | { | ||
63 | continue; | 73 | continue; |
64 | } | 74 | } |
65 | match child { | 75 | match child { |
@@ -127,6 +137,11 @@ impl<'a, Q: Querier> TtTreeSink<'a, Q> { | |||
127 | 137 | ||
128 | impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { | 138 | impl<'a, Q: Querier> TreeSink for TtTreeSink<'a, Q> { |
129 | fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { | 139 | fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { |
140 | if kind == L_DOLLAR || kind == R_DOLLAR { | ||
141 | self.token_pos += n_tokens as usize; | ||
142 | return; | ||
143 | } | ||
144 | |||
130 | for _ in 0..n_tokens { | 145 | for _ in 0..n_tokens { |
131 | self.buf += &self.src_querier.token(self.token_pos).1; | 146 | self.buf += &self.src_querier.token(self.token_pos).1; |
132 | self.token_pos += 1; | 147 | self.token_pos += 1; |
@@ -176,19 +191,19 @@ mod tests { | |||
176 | 191 | ||
177 | let query = tt_src.querier(); | 192 | let query = tt_src.querier(); |
178 | 193 | ||
179 | // [{] | 194 | // [${] |
180 | // [let] [a] [=] ['c'] [;] | 195 | // [let] [a] [=] ['c'] [;] |
181 | assert_eq!(query.token(1 + 3).1, "'c'"); | 196 | assert_eq!(query.token(2 + 3).1, "'c'"); |
182 | assert_eq!(query.token(1 + 3).0, CHAR); | 197 | assert_eq!(query.token(2 + 3).0, CHAR); |
183 | // [let] [c] [=] [1000] [;] | 198 | // [let] [c] [=] [1000] [;] |
184 | assert_eq!(query.token(1 + 5 + 3).1, "1000"); | 199 | assert_eq!(query.token(2 + 5 + 3).1, "1000"); |
185 | assert_eq!(query.token(1 + 5 + 3).0, INT_NUMBER); | 200 | assert_eq!(query.token(2 + 5 + 3).0, INT_NUMBER); |
186 | // [let] [f] [=] [12E+99_f64] [;] | 201 | // [let] [f] [=] [12E+99_f64] [;] |
187 | assert_eq!(query.token(1 + 10 + 3).1, "12E+99_f64"); | 202 | assert_eq!(query.token(2 + 10 + 3).1, "12E+99_f64"); |
188 | assert_eq!(query.token(1 + 10 + 3).0, FLOAT_NUMBER); | 203 | assert_eq!(query.token(2 + 10 + 3).0, FLOAT_NUMBER); |
189 | 204 | ||
190 | // [let] [s] [=] ["rust1"] [;] | 205 | // [let] [s] [=] ["rust1"] [;] |
191 | assert_eq!(query.token(1 + 15 + 3).1, "\"rust1\""); | 206 | assert_eq!(query.token(2 + 15 + 3).1, "\"rust1\""); |
192 | assert_eq!(query.token(1 + 15 + 3).0, STRING); | 207 | assert_eq!(query.token(2 + 15 + 3).0, STRING); |
193 | } | 208 | } |
194 | } | 209 | } |