diff options
author | Aleksey Kladov <[email protected]> | 2019-09-17 00:06:14 +0100 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2019-09-17 00:06:14 +0100 |
commit | 37ef8927c373b8eadd63edc1f70055428c49290e (patch) | |
tree | b8bee4dd75c07f55b9f0d7e86380aba904557c3a /crates/ra_mbe | |
parent | ba583091e60553633dd3cc9ab37a1d9f64827a1e (diff) |
split mbe expander code into two modules
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r-- | crates/ra_mbe/src/mbe_expander.rs | 424 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_expander/matcher.rs | 215 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_expander/transcriber.rs | 227 |
3 files changed, 448 insertions, 418 deletions
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 78df96880..15d9d83e2 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs | |||
@@ -2,10 +2,11 @@ | |||
2 | //! `tt::TokenTree` representing an argument of macro invocation, and produces a | 2 | //! `tt::TokenTree` representing an argument of macro invocation, and produces a |
3 | //! `tt::TokenTree` for the result of the expansion. | 3 | //! `tt::TokenTree` for the result of the expansion. |
4 | 4 | ||
5 | use ra_parser::FragmentKind::*; | 5 | mod matcher; |
6 | mod transcriber; | ||
7 | |||
6 | use ra_syntax::SmolStr; | 8 | use ra_syntax::SmolStr; |
7 | use rustc_hash::FxHashMap; | 9 | use rustc_hash::FxHashMap; |
8 | use tt::TokenId; | ||
9 | 10 | ||
10 | use crate::tt_cursor::TtCursor; | 11 | use crate::tt_cursor::TtCursor; |
11 | use crate::ExpandError; | 12 | use crate::ExpandError; |
@@ -19,14 +20,12 @@ pub(crate) fn expand( | |||
19 | 20 | ||
20 | fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { | 21 | fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { |
21 | let mut input = TtCursor::new(input); | 22 | let mut input = TtCursor::new(input); |
22 | let bindings = match_lhs(&rule.lhs, &mut input)?; | 23 | let bindings = matcher::match_lhs(&rule.lhs, &mut input)?; |
23 | if !input.is_eof() { | 24 | if !input.is_eof() { |
24 | return Err(ExpandError::UnexpectedToken); | 25 | return Err(ExpandError::UnexpectedToken); |
25 | } | 26 | } |
26 | 27 | let res = transcriber::transcribe(&bindings, &rule.rhs)?; | |
27 | let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; | 28 | Ok(res) |
28 | |||
29 | expand_subtree(&rule.rhs, &mut ctx) | ||
30 | } | 29 | } |
31 | 30 | ||
32 | /// The actual algorithm for expansion is not too hard, but is pretty tricky. | 31 | /// The actual algorithm for expansion is not too hard, but is pretty tricky. |
@@ -95,403 +94,6 @@ enum Fragment { | |||
95 | Ast(tt::TokenTree), | 94 | Ast(tt::TokenTree), |
96 | } | 95 | } |
97 | 96 | ||
98 | impl Bindings { | ||
99 | fn push_optional(&mut self, name: &SmolStr) { | ||
100 | // FIXME: Do we have a better way to represent an empty token ? | ||
101 | // Insert an empty subtree for empty token | ||
102 | let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: vec![] }.into(); | ||
103 | self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); | ||
104 | } | ||
105 | |||
106 | fn push_empty(&mut self, name: &SmolStr) { | ||
107 | self.inner.insert(name.clone(), Binding::Empty); | ||
108 | } | ||
109 | |||
110 | fn contains(&self, name: &SmolStr) -> bool { | ||
111 | self.inner.contains_key(name) | ||
112 | } | ||
113 | |||
114 | fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { | ||
115 | let mut b = self.inner.get(name).ok_or_else(|| { | ||
116 | ExpandError::BindingError(format!("could not find binding `{}`", name)) | ||
117 | })?; | ||
118 | for &idx in nesting.iter() { | ||
119 | b = match b { | ||
120 | Binding::Fragment(_) => break, | ||
121 | Binding::Nested(bs) => bs.get(idx).ok_or_else(|| { | ||
122 | ExpandError::BindingError(format!("could not find nested binding `{}`", name)) | ||
123 | })?, | ||
124 | Binding::Empty => { | ||
125 | return Err(ExpandError::BindingError(format!( | ||
126 | "could not find empty binding `{}`", | ||
127 | name | ||
128 | ))) | ||
129 | } | ||
130 | }; | ||
131 | } | ||
132 | match b { | ||
133 | Binding::Fragment(it) => Ok(it), | ||
134 | Binding::Nested(_) => Err(ExpandError::BindingError(format!( | ||
135 | "expected simple binding, found nested binding `{}`", | ||
136 | name | ||
137 | ))), | ||
138 | Binding::Empty => Err(ExpandError::BindingError(format!( | ||
139 | "expected simple binding, found empty binding `{}`", | ||
140 | name | ||
141 | ))), | ||
142 | } | ||
143 | } | ||
144 | |||
145 | fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> { | ||
146 | for (key, value) in nested.inner { | ||
147 | if !self.inner.contains_key(&key) { | ||
148 | self.inner.insert(key.clone(), Binding::Nested(Vec::new())); | ||
149 | } | ||
150 | match self.inner.get_mut(&key) { | ||
151 | Some(Binding::Nested(it)) => { | ||
152 | // insert empty nested bindings before this one | ||
153 | while it.len() < idx { | ||
154 | it.push(Binding::Nested(vec![])); | ||
155 | } | ||
156 | it.push(value); | ||
157 | } | ||
158 | _ => { | ||
159 | return Err(ExpandError::BindingError(format!( | ||
160 | "could not find binding `{}`", | ||
161 | key | ||
162 | ))); | ||
163 | } | ||
164 | } | ||
165 | } | ||
166 | Ok(()) | ||
167 | } | ||
168 | |||
169 | fn merge(&mut self, nested: Bindings) { | ||
170 | self.inner.extend(nested.inner); | ||
171 | } | ||
172 | } | ||
173 | |||
174 | fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> { | ||
175 | let mut res = vec![]; | ||
176 | |||
177 | for tkn in subtree.token_trees.iter() { | ||
178 | match tkn { | ||
179 | crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => { | ||
180 | res.push(text.clone()); | ||
181 | } | ||
182 | crate::TokenTree::Subtree(subtree) => { | ||
183 | res.extend(collect_vars(subtree)); | ||
184 | } | ||
185 | crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => { | ||
186 | res.extend(collect_vars(subtree)); | ||
187 | } | ||
188 | _ => {} | ||
189 | } | ||
190 | } | ||
191 | |||
192 | res | ||
193 | } | ||
194 | |||
195 | fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> { | ||
196 | let mut res = Bindings::default(); | ||
197 | for pat in pattern.token_trees.iter() { | ||
198 | match pat { | ||
199 | crate::TokenTree::Leaf(leaf) => match leaf { | ||
200 | crate::Leaf::Var(crate::Var { text, kind }) => { | ||
201 | let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; | ||
202 | match match_meta_var(kind.as_str(), input)? { | ||
203 | Some(fragment) => { | ||
204 | res.inner.insert(text.clone(), Binding::Fragment(fragment)); | ||
205 | } | ||
206 | None => res.push_optional(text), | ||
207 | } | ||
208 | } | ||
209 | crate::Leaf::Punct(punct) => { | ||
210 | if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) { | ||
211 | return Err(ExpandError::UnexpectedToken); | ||
212 | } | ||
213 | } | ||
214 | crate::Leaf::Ident(ident) => { | ||
215 | if input.eat_ident().map(|i| &i.text) != Some(&ident.text) { | ||
216 | return Err(ExpandError::UnexpectedToken); | ||
217 | } | ||
218 | } | ||
219 | crate::Leaf::Literal(literal) => { | ||
220 | if input.eat_literal().map(|i| &i.text) != Some(&literal.text) { | ||
221 | return Err(ExpandError::UnexpectedToken); | ||
222 | } | ||
223 | } | ||
224 | }, | ||
225 | crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => { | ||
226 | // Dirty hack to make macro-expansion terminate. | ||
227 | // This should be replaced by a propper macro-by-example implementation | ||
228 | let mut limit = 65536; | ||
229 | let mut counter = 0; | ||
230 | |||
231 | let mut memento = input.save(); | ||
232 | |||
233 | loop { | ||
234 | match match_lhs(subtree, input) { | ||
235 | Ok(nested) => { | ||
236 | limit -= 1; | ||
237 | if limit == 0 { | ||
238 | log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); | ||
239 | break; | ||
240 | } | ||
241 | |||
242 | memento = input.save(); | ||
243 | res.push_nested(counter, nested)?; | ||
244 | counter += 1; | ||
245 | if counter == 1 { | ||
246 | if let crate::RepeatKind::ZeroOrOne = kind { | ||
247 | break; | ||
248 | } | ||
249 | } | ||
250 | |||
251 | if let Some(separator) = separator { | ||
252 | if !input | ||
253 | .eat_seperator() | ||
254 | .map(|sep| sep == *separator) | ||
255 | .unwrap_or(false) | ||
256 | { | ||
257 | input.rollback(memento); | ||
258 | break; | ||
259 | } | ||
260 | } | ||
261 | } | ||
262 | Err(_) => { | ||
263 | input.rollback(memento); | ||
264 | break; | ||
265 | } | ||
266 | } | ||
267 | } | ||
268 | |||
269 | match kind { | ||
270 | crate::RepeatKind::OneOrMore if counter == 0 => { | ||
271 | return Err(ExpandError::UnexpectedToken); | ||
272 | } | ||
273 | _ if counter == 0 => { | ||
274 | // Collect all empty variables in subtrees | ||
275 | collect_vars(subtree).iter().for_each(|s| res.push_empty(s)); | ||
276 | } | ||
277 | _ => {} | ||
278 | } | ||
279 | } | ||
280 | crate::TokenTree::Subtree(subtree) => { | ||
281 | let input_subtree = | ||
282 | input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?; | ||
283 | if subtree.delimiter != input_subtree.delimiter { | ||
284 | return Err(ExpandError::UnexpectedToken); | ||
285 | } | ||
286 | |||
287 | let mut input = TtCursor::new(input_subtree); | ||
288 | let bindings = match_lhs(&subtree, &mut input)?; | ||
289 | if !input.is_eof() { | ||
290 | return Err(ExpandError::UnexpectedToken); | ||
291 | } | ||
292 | |||
293 | res.merge(bindings); | ||
294 | } | ||
295 | } | ||
296 | } | ||
297 | Ok(res) | ||
298 | } | ||
299 | |||
300 | fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> { | ||
301 | let fragment = match kind { | ||
302 | "path" => Path, | ||
303 | "expr" => Expr, | ||
304 | "ty" => Type, | ||
305 | "pat" => Pattern, | ||
306 | "stmt" => Statement, | ||
307 | "block" => Block, | ||
308 | "meta" => MetaItem, | ||
309 | "item" => Item, | ||
310 | _ => { | ||
311 | let tt = match kind { | ||
312 | "ident" => { | ||
313 | let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
314 | tt::Leaf::from(ident).into() | ||
315 | } | ||
316 | "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(), | ||
317 | "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(), | ||
318 | "literal" => { | ||
319 | let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
320 | tt::Leaf::from(literal).into() | ||
321 | } | ||
322 | // `vis` is optional | ||
323 | "vis" => match input.try_eat_vis() { | ||
324 | Some(vis) => vis, | ||
325 | None => return Ok(None), | ||
326 | }, | ||
327 | _ => return Err(ExpandError::UnexpectedToken), | ||
328 | }; | ||
329 | return Ok(Some(Fragment::Tokens(tt))); | ||
330 | } | ||
331 | }; | ||
332 | let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?; | ||
333 | let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; | ||
334 | Ok(Some(fragment)) | ||
335 | } | ||
336 | |||
337 | #[derive(Debug)] | ||
338 | struct ExpandCtx<'a> { | ||
339 | bindings: &'a Bindings, | ||
340 | nesting: Vec<usize>, | ||
341 | var_expanded: bool, | ||
342 | } | ||
343 | |||
344 | fn expand_subtree( | ||
345 | template: &crate::Subtree, | ||
346 | ctx: &mut ExpandCtx, | ||
347 | ) -> Result<tt::Subtree, ExpandError> { | ||
348 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | ||
349 | for tt in template.token_trees.iter() { | ||
350 | let tt = expand_tt(tt, ctx)?; | ||
351 | push_fragment(&mut buf, tt); | ||
352 | } | ||
353 | |||
354 | Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) | ||
355 | } | ||
356 | |||
357 | fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> { | ||
358 | let res: tt::TokenTree = match template { | ||
359 | crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), | ||
360 | crate::TokenTree::Repeat(repeat) => { | ||
361 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | ||
362 | ctx.nesting.push(0); | ||
363 | // Dirty hack to make macro-expansion terminate. | ||
364 | // This should be replaced by a propper macro-by-example implementation | ||
365 | let mut limit = 65536; | ||
366 | let mut has_seps = 0; | ||
367 | let mut counter = 0; | ||
368 | |||
369 | // We store the old var expanded value, and restore it later | ||
370 | // It is because before this `$repeat`, | ||
371 | // it is possible some variables already expanad in the same subtree | ||
372 | // | ||
373 | // `some_var_expanded` keep check if the deeper subtree has expanded variables | ||
374 | let mut some_var_expanded = false; | ||
375 | let old_var_expanded = ctx.var_expanded; | ||
376 | ctx.var_expanded = false; | ||
377 | |||
378 | while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { | ||
379 | // if no var expanded in the child, we count it as a fail | ||
380 | if !ctx.var_expanded { | ||
381 | break; | ||
382 | } | ||
383 | |||
384 | // Reset `ctx.var_expandeded` to see if there is other expanded variable | ||
385 | // in the next matching | ||
386 | some_var_expanded = true; | ||
387 | ctx.var_expanded = false; | ||
388 | |||
389 | counter += 1; | ||
390 | limit -= 1; | ||
391 | if limit == 0 { | ||
392 | log::warn!( | ||
393 | "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", | ||
394 | template, | ||
395 | ctx | ||
396 | ); | ||
397 | break; | ||
398 | } | ||
399 | |||
400 | let idx = ctx.nesting.pop().unwrap(); | ||
401 | ctx.nesting.push(idx + 1); | ||
402 | push_subtree(&mut buf, t); | ||
403 | |||
404 | if let Some(ref sep) = repeat.separator { | ||
405 | match sep { | ||
406 | crate::Separator::Ident(ident) => { | ||
407 | has_seps = 1; | ||
408 | buf.push(tt::Leaf::from(ident.clone()).into()); | ||
409 | } | ||
410 | crate::Separator::Literal(lit) => { | ||
411 | has_seps = 1; | ||
412 | buf.push(tt::Leaf::from(lit.clone()).into()); | ||
413 | } | ||
414 | |||
415 | crate::Separator::Puncts(puncts) => { | ||
416 | has_seps = puncts.len(); | ||
417 | for punct in puncts { | ||
418 | buf.push(tt::Leaf::from(*punct).into()); | ||
419 | } | ||
420 | } | ||
421 | } | ||
422 | } | ||
423 | |||
424 | if let crate::RepeatKind::ZeroOrOne = repeat.kind { | ||
425 | break; | ||
426 | } | ||
427 | } | ||
428 | |||
429 | // Restore the `var_expanded` by combining old one and the new one | ||
430 | ctx.var_expanded = some_var_expanded || old_var_expanded; | ||
431 | |||
432 | ctx.nesting.pop().unwrap(); | ||
433 | for _ in 0..has_seps { | ||
434 | buf.pop(); | ||
435 | } | ||
436 | |||
437 | if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { | ||
438 | return Err(ExpandError::UnexpectedToken); | ||
439 | } | ||
440 | |||
441 | // Check if it is a single token subtree without any delimiter | ||
442 | // e.g {Delimiter:None> ['>'] /Delimiter:None>} | ||
443 | tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() | ||
444 | } | ||
445 | crate::TokenTree::Leaf(leaf) => match leaf { | ||
446 | crate::Leaf::Ident(ident) => { | ||
447 | tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() }) | ||
448 | .into() | ||
449 | } | ||
450 | crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(), | ||
451 | crate::Leaf::Var(v) => { | ||
452 | if v.text == "crate" { | ||
453 | // FIXME: Properly handle $crate token | ||
454 | tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() }) | ||
455 | .into() | ||
456 | } else if !ctx.bindings.contains(&v.text) { | ||
457 | // Note that it is possible to have a `$var` inside a macro which is not bound. | ||
458 | // For example: | ||
459 | // ``` | ||
460 | // macro_rules! foo { | ||
461 | // ($a:ident, $b:ident, $c:tt) => { | ||
462 | // macro_rules! bar { | ||
463 | // ($bi:ident) => { | ||
464 | // fn $bi() -> u8 {$c} | ||
465 | // } | ||
466 | // } | ||
467 | // } | ||
468 | // ``` | ||
469 | // We just treat it a normal tokens | ||
470 | tt::Subtree { | ||
471 | delimiter: tt::Delimiter::None, | ||
472 | token_trees: vec![ | ||
473 | tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }) | ||
474 | .into(), | ||
475 | tt::Leaf::from(tt::Ident { | ||
476 | text: v.text.clone(), | ||
477 | id: TokenId::unspecified(), | ||
478 | }) | ||
479 | .into(), | ||
480 | ], | ||
481 | } | ||
482 | .into() | ||
483 | } else { | ||
484 | let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); | ||
485 | ctx.var_expanded = true; | ||
486 | return Ok(fragment); | ||
487 | } | ||
488 | } | ||
489 | crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(), | ||
490 | }, | ||
491 | }; | ||
492 | Ok(Fragment::Tokens(res)) | ||
493 | } | ||
494 | |||
495 | #[cfg(test)] | 97 | #[cfg(test)] |
496 | mod tests { | 98 | mod tests { |
497 | use ra_syntax::{ast, AstNode}; | 99 | use ra_syntax::{ast, AstNode}; |
@@ -562,17 +164,3 @@ mod tests { | |||
562 | expand_rule(&rules.rules[0], &invocation_tt) | 164 | expand_rule(&rules.rules[0], &invocation_tt) |
563 | } | 165 | } |
564 | } | 166 | } |
565 | |||
566 | fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { | ||
567 | match fragment { | ||
568 | Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), | ||
569 | Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), | ||
570 | } | ||
571 | } | ||
572 | |||
573 | fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) { | ||
574 | match tt.delimiter { | ||
575 | tt::Delimiter::None => buf.extend(tt.token_trees), | ||
576 | _ => buf.push(tt.into()), | ||
577 | } | ||
578 | } | ||
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs new file mode 100644 index 000000000..100a3b0e0 --- /dev/null +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs | |||
@@ -0,0 +1,215 @@ | |||
1 | use crate::{ | ||
2 | mbe_expander::{Binding, Bindings, Fragment}, | ||
3 | tt_cursor::TtCursor, | ||
4 | ExpandError, | ||
5 | }; | ||
6 | |||
7 | use ra_parser::FragmentKind::*; | ||
8 | use ra_syntax::SmolStr; | ||
9 | |||
10 | impl Bindings { | ||
11 | fn push_optional(&mut self, name: &SmolStr) { | ||
12 | // FIXME: Do we have a better way to represent an empty token ? | ||
13 | // Insert an empty subtree for empty token | ||
14 | let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: vec![] }.into(); | ||
15 | self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt))); | ||
16 | } | ||
17 | |||
18 | fn push_empty(&mut self, name: &SmolStr) { | ||
19 | self.inner.insert(name.clone(), Binding::Empty); | ||
20 | } | ||
21 | |||
22 | fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> { | ||
23 | for (key, value) in nested.inner { | ||
24 | if !self.inner.contains_key(&key) { | ||
25 | self.inner.insert(key.clone(), Binding::Nested(Vec::new())); | ||
26 | } | ||
27 | match self.inner.get_mut(&key) { | ||
28 | Some(Binding::Nested(it)) => { | ||
29 | // insert empty nested bindings before this one | ||
30 | while it.len() < idx { | ||
31 | it.push(Binding::Nested(vec![])); | ||
32 | } | ||
33 | it.push(value); | ||
34 | } | ||
35 | _ => { | ||
36 | return Err(ExpandError::BindingError(format!( | ||
37 | "could not find binding `{}`", | ||
38 | key | ||
39 | ))); | ||
40 | } | ||
41 | } | ||
42 | } | ||
43 | Ok(()) | ||
44 | } | ||
45 | |||
46 | fn merge(&mut self, nested: Bindings) { | ||
47 | self.inner.extend(nested.inner); | ||
48 | } | ||
49 | } | ||
50 | |||
51 | pub(super) fn match_lhs( | ||
52 | pattern: &crate::Subtree, | ||
53 | input: &mut TtCursor, | ||
54 | ) -> Result<Bindings, ExpandError> { | ||
55 | let mut res = Bindings::default(); | ||
56 | for pat in pattern.token_trees.iter() { | ||
57 | match pat { | ||
58 | crate::TokenTree::Leaf(leaf) => match leaf { | ||
59 | crate::Leaf::Var(crate::Var { text, kind }) => { | ||
60 | let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; | ||
61 | match match_meta_var(kind.as_str(), input)? { | ||
62 | Some(fragment) => { | ||
63 | res.inner.insert(text.clone(), Binding::Fragment(fragment)); | ||
64 | } | ||
65 | None => res.push_optional(text), | ||
66 | } | ||
67 | } | ||
68 | crate::Leaf::Punct(punct) => { | ||
69 | if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) { | ||
70 | return Err(ExpandError::UnexpectedToken); | ||
71 | } | ||
72 | } | ||
73 | crate::Leaf::Ident(ident) => { | ||
74 | if input.eat_ident().map(|i| &i.text) != Some(&ident.text) { | ||
75 | return Err(ExpandError::UnexpectedToken); | ||
76 | } | ||
77 | } | ||
78 | crate::Leaf::Literal(literal) => { | ||
79 | if input.eat_literal().map(|i| &i.text) != Some(&literal.text) { | ||
80 | return Err(ExpandError::UnexpectedToken); | ||
81 | } | ||
82 | } | ||
83 | }, | ||
84 | crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => { | ||
85 | // Dirty hack to make macro-expansion terminate. | ||
86 | // This should be replaced by a propper macro-by-example implementation | ||
87 | let mut limit = 65536; | ||
88 | let mut counter = 0; | ||
89 | |||
90 | let mut memento = input.save(); | ||
91 | |||
92 | loop { | ||
93 | match match_lhs(subtree, input) { | ||
94 | Ok(nested) => { | ||
95 | limit -= 1; | ||
96 | if limit == 0 { | ||
97 | log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); | ||
98 | break; | ||
99 | } | ||
100 | |||
101 | memento = input.save(); | ||
102 | res.push_nested(counter, nested)?; | ||
103 | counter += 1; | ||
104 | if counter == 1 { | ||
105 | if let crate::RepeatKind::ZeroOrOne = kind { | ||
106 | break; | ||
107 | } | ||
108 | } | ||
109 | |||
110 | if let Some(separator) = separator { | ||
111 | if !input | ||
112 | .eat_seperator() | ||
113 | .map(|sep| sep == *separator) | ||
114 | .unwrap_or(false) | ||
115 | { | ||
116 | input.rollback(memento); | ||
117 | break; | ||
118 | } | ||
119 | } | ||
120 | } | ||
121 | Err(_) => { | ||
122 | input.rollback(memento); | ||
123 | break; | ||
124 | } | ||
125 | } | ||
126 | } | ||
127 | |||
128 | match kind { | ||
129 | crate::RepeatKind::OneOrMore if counter == 0 => { | ||
130 | return Err(ExpandError::UnexpectedToken); | ||
131 | } | ||
132 | _ if counter == 0 => { | ||
133 | // Collect all empty variables in subtrees | ||
134 | collect_vars(subtree).iter().for_each(|s| res.push_empty(s)); | ||
135 | } | ||
136 | _ => {} | ||
137 | } | ||
138 | } | ||
139 | crate::TokenTree::Subtree(subtree) => { | ||
140 | let input_subtree = | ||
141 | input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?; | ||
142 | if subtree.delimiter != input_subtree.delimiter { | ||
143 | return Err(ExpandError::UnexpectedToken); | ||
144 | } | ||
145 | |||
146 | let mut input = TtCursor::new(input_subtree); | ||
147 | let bindings = match_lhs(&subtree, &mut input)?; | ||
148 | if !input.is_eof() { | ||
149 | return Err(ExpandError::UnexpectedToken); | ||
150 | } | ||
151 | |||
152 | res.merge(bindings); | ||
153 | } | ||
154 | } | ||
155 | } | ||
156 | Ok(res) | ||
157 | } | ||
158 | |||
159 | fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> { | ||
160 | let fragment = match kind { | ||
161 | "path" => Path, | ||
162 | "expr" => Expr, | ||
163 | "ty" => Type, | ||
164 | "pat" => Pattern, | ||
165 | "stmt" => Statement, | ||
166 | "block" => Block, | ||
167 | "meta" => MetaItem, | ||
168 | "item" => Item, | ||
169 | _ => { | ||
170 | let tt = match kind { | ||
171 | "ident" => { | ||
172 | let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
173 | tt::Leaf::from(ident).into() | ||
174 | } | ||
175 | "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(), | ||
176 | "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(), | ||
177 | "literal" => { | ||
178 | let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); | ||
179 | tt::Leaf::from(literal).into() | ||
180 | } | ||
181 | // `vis` is optional | ||
182 | "vis" => match input.try_eat_vis() { | ||
183 | Some(vis) => vis, | ||
184 | None => return Ok(None), | ||
185 | }, | ||
186 | _ => return Err(ExpandError::UnexpectedToken), | ||
187 | }; | ||
188 | return Ok(Some(Fragment::Tokens(tt))); | ||
189 | } | ||
190 | }; | ||
191 | let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?; | ||
192 | let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; | ||
193 | Ok(Some(fragment)) | ||
194 | } | ||
195 | |||
196 | fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> { | ||
197 | let mut res = Vec::new(); | ||
198 | |||
199 | for tkn in subtree.token_trees.iter() { | ||
200 | match tkn { | ||
201 | crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => { | ||
202 | res.push(text.clone()); | ||
203 | } | ||
204 | crate::TokenTree::Subtree(subtree) => { | ||
205 | res.extend(collect_vars(subtree)); | ||
206 | } | ||
207 | crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => { | ||
208 | res.extend(collect_vars(subtree)); | ||
209 | } | ||
210 | _ => {} | ||
211 | } | ||
212 | } | ||
213 | |||
214 | res | ||
215 | } | ||
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs new file mode 100644 index 000000000..a3df1b7de --- /dev/null +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs | |||
@@ -0,0 +1,227 @@ | |||
1 | use ra_syntax::SmolStr; | ||
2 | |||
3 | use crate::{ | ||
4 | mbe_expander::{Binding, Bindings, Fragment}, | ||
5 | ExpandError, | ||
6 | }; | ||
7 | |||
8 | impl Bindings { | ||
9 | fn contains(&self, name: &SmolStr) -> bool { | ||
10 | self.inner.contains_key(name) | ||
11 | } | ||
12 | |||
13 | fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { | ||
14 | let mut b = self.inner.get(name).ok_or_else(|| { | ||
15 | ExpandError::BindingError(format!("could not find binding `{}`", name)) | ||
16 | })?; | ||
17 | for &idx in nesting.iter() { | ||
18 | b = match b { | ||
19 | Binding::Fragment(_) => break, | ||
20 | Binding::Nested(bs) => bs.get(idx).ok_or_else(|| { | ||
21 | ExpandError::BindingError(format!("could not find nested binding `{}`", name)) | ||
22 | })?, | ||
23 | Binding::Empty => { | ||
24 | return Err(ExpandError::BindingError(format!( | ||
25 | "could not find empty binding `{}`", | ||
26 | name | ||
27 | ))) | ||
28 | } | ||
29 | }; | ||
30 | } | ||
31 | match b { | ||
32 | Binding::Fragment(it) => Ok(it), | ||
33 | Binding::Nested(_) => Err(ExpandError::BindingError(format!( | ||
34 | "expected simple binding, found nested binding `{}`", | ||
35 | name | ||
36 | ))), | ||
37 | Binding::Empty => Err(ExpandError::BindingError(format!( | ||
38 | "expected simple binding, found empty binding `{}`", | ||
39 | name | ||
40 | ))), | ||
41 | } | ||
42 | } | ||
43 | } | ||
44 | |||
45 | pub(super) fn transcribe( | ||
46 | bindings: &Bindings, | ||
47 | template: &crate::Subtree, | ||
48 | ) -> Result<tt::Subtree, ExpandError> { | ||
49 | let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; | ||
50 | expand_subtree(template, &mut ctx) | ||
51 | } | ||
52 | |||
53 | #[derive(Debug)] | ||
54 | struct ExpandCtx<'a> { | ||
55 | bindings: &'a Bindings, | ||
56 | nesting: Vec<usize>, | ||
57 | var_expanded: bool, | ||
58 | } | ||
59 | |||
60 | fn expand_subtree( | ||
61 | template: &crate::Subtree, | ||
62 | ctx: &mut ExpandCtx, | ||
63 | ) -> Result<tt::Subtree, ExpandError> { | ||
64 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | ||
65 | for tt in template.token_trees.iter() { | ||
66 | let tt = expand_tt(tt, ctx)?; | ||
67 | push_fragment(&mut buf, tt); | ||
68 | } | ||
69 | |||
70 | Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) | ||
71 | } | ||
72 | |||
73 | fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> { | ||
74 | let res: tt::TokenTree = match template { | ||
75 | crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), | ||
76 | crate::TokenTree::Repeat(repeat) => { | ||
77 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | ||
78 | ctx.nesting.push(0); | ||
79 | // Dirty hack to make macro-expansion terminate. | ||
80 | // This should be replaced by a propper macro-by-example implementation | ||
81 | let mut limit = 65536; | ||
82 | let mut has_seps = 0; | ||
83 | let mut counter = 0; | ||
84 | |||
85 | // We store the old var expanded value, and restore it later | ||
86 | // It is because before this `$repeat`, | ||
87 | // it is possible some variables already expanad in the same subtree | ||
88 | // | ||
89 | // `some_var_expanded` keep check if the deeper subtree has expanded variables | ||
90 | let mut some_var_expanded = false; | ||
91 | let old_var_expanded = ctx.var_expanded; | ||
92 | ctx.var_expanded = false; | ||
93 | |||
94 | while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { | ||
95 | // if no var expanded in the child, we count it as a fail | ||
96 | if !ctx.var_expanded { | ||
97 | break; | ||
98 | } | ||
99 | |||
100 | // Reset `ctx.var_expandeded` to see if there is other expanded variable | ||
101 | // in the next matching | ||
102 | some_var_expanded = true; | ||
103 | ctx.var_expanded = false; | ||
104 | |||
105 | counter += 1; | ||
106 | limit -= 1; | ||
107 | if limit == 0 { | ||
108 | log::warn!( | ||
109 | "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", | ||
110 | template, | ||
111 | ctx | ||
112 | ); | ||
113 | break; | ||
114 | } | ||
115 | |||
116 | let idx = ctx.nesting.pop().unwrap(); | ||
117 | ctx.nesting.push(idx + 1); | ||
118 | push_subtree(&mut buf, t); | ||
119 | |||
120 | if let Some(ref sep) = repeat.separator { | ||
121 | match sep { | ||
122 | crate::Separator::Ident(ident) => { | ||
123 | has_seps = 1; | ||
124 | buf.push(tt::Leaf::from(ident.clone()).into()); | ||
125 | } | ||
126 | crate::Separator::Literal(lit) => { | ||
127 | has_seps = 1; | ||
128 | buf.push(tt::Leaf::from(lit.clone()).into()); | ||
129 | } | ||
130 | |||
131 | crate::Separator::Puncts(puncts) => { | ||
132 | has_seps = puncts.len(); | ||
133 | for punct in puncts { | ||
134 | buf.push(tt::Leaf::from(*punct).into()); | ||
135 | } | ||
136 | } | ||
137 | } | ||
138 | } | ||
139 | |||
140 | if let crate::RepeatKind::ZeroOrOne = repeat.kind { | ||
141 | break; | ||
142 | } | ||
143 | } | ||
144 | |||
145 | // Restore the `var_expanded` by combining old one and the new one | ||
146 | ctx.var_expanded = some_var_expanded || old_var_expanded; | ||
147 | |||
148 | ctx.nesting.pop().unwrap(); | ||
149 | for _ in 0..has_seps { | ||
150 | buf.pop(); | ||
151 | } | ||
152 | |||
153 | if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { | ||
154 | return Err(ExpandError::UnexpectedToken); | ||
155 | } | ||
156 | |||
157 | // Check if it is a single token subtree without any delimiter | ||
158 | // e.g {Delimiter:None> ['>'] /Delimiter:None>} | ||
159 | tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() | ||
160 | } | ||
161 | crate::TokenTree::Leaf(leaf) => match leaf { | ||
162 | crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident { | ||
163 | text: ident.text.clone(), | ||
164 | id: tt::TokenId::unspecified(), | ||
165 | }) | ||
166 | .into(), | ||
167 | crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(), | ||
168 | crate::Leaf::Var(v) => { | ||
169 | if v.text == "crate" { | ||
170 | // FIXME: Properly handle $crate token | ||
171 | tt::Leaf::from(tt::Ident { | ||
172 | text: "$crate".into(), | ||
173 | id: tt::TokenId::unspecified(), | ||
174 | }) | ||
175 | .into() | ||
176 | } else if !ctx.bindings.contains(&v.text) { | ||
177 | // Note that it is possible to have a `$var` inside a macro which is not bound. | ||
178 | // For example: | ||
179 | // ``` | ||
180 | // macro_rules! foo { | ||
181 | // ($a:ident, $b:ident, $c:tt) => { | ||
182 | // macro_rules! bar { | ||
183 | // ($bi:ident) => { | ||
184 | // fn $bi() -> u8 {$c} | ||
185 | // } | ||
186 | // } | ||
187 | // } | ||
188 | // ``` | ||
189 | // We just treat it a normal tokens | ||
190 | tt::Subtree { | ||
191 | delimiter: tt::Delimiter::None, | ||
192 | token_trees: vec![ | ||
193 | tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }) | ||
194 | .into(), | ||
195 | tt::Leaf::from(tt::Ident { | ||
196 | text: v.text.clone(), | ||
197 | id: tt::TokenId::unspecified(), | ||
198 | }) | ||
199 | .into(), | ||
200 | ], | ||
201 | } | ||
202 | .into() | ||
203 | } else { | ||
204 | let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); | ||
205 | ctx.var_expanded = true; | ||
206 | return Ok(fragment); | ||
207 | } | ||
208 | } | ||
209 | crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(), | ||
210 | }, | ||
211 | }; | ||
212 | Ok(Fragment::Tokens(res)) | ||
213 | } | ||
214 | |||
215 | fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { | ||
216 | match fragment { | ||
217 | Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), | ||
218 | Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), | ||
219 | } | ||
220 | } | ||
221 | |||
222 | fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) { | ||
223 | match tt.delimiter { | ||
224 | tt::Delimiter::None => buf.extend(tt.token_trees), | ||
225 | _ => buf.push(tt.into()), | ||
226 | } | ||
227 | } | ||