aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2019-09-17 00:06:38 +0100
committerGitHub <[email protected]>2019-09-17 00:06:38 +0100
commit8eb2697b7d2a98c952b3acd1711829a13e13cab1 (patch)
treeb8bee4dd75c07f55b9f0d7e86380aba904557c3a /crates/ra_mbe
parentba583091e60553633dd3cc9ab37a1d9f64827a1e (diff)
parent37ef8927c373b8eadd63edc1f70055428c49290e (diff)
Merge #1855
1855: split mbe expander code into two modules r=matklad a=matklad Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs424
-rw-r--r--crates/ra_mbe/src/mbe_expander/matcher.rs215
-rw-r--r--crates/ra_mbe/src/mbe_expander/transcriber.rs227
3 files changed, 448 insertions, 418 deletions
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 78df96880..15d9d83e2 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -2,10 +2,11 @@
2//! `tt::TokenTree` representing an argument of macro invocation, and produces a 2//! `tt::TokenTree` representing an argument of macro invocation, and produces a
3//! `tt::TokenTree` for the result of the expansion. 3//! `tt::TokenTree` for the result of the expansion.
4 4
5use ra_parser::FragmentKind::*; 5mod matcher;
6mod transcriber;
7
6use ra_syntax::SmolStr; 8use ra_syntax::SmolStr;
7use rustc_hash::FxHashMap; 9use rustc_hash::FxHashMap;
8use tt::TokenId;
9 10
10use crate::tt_cursor::TtCursor; 11use crate::tt_cursor::TtCursor;
11use crate::ExpandError; 12use crate::ExpandError;
@@ -19,14 +20,12 @@ pub(crate) fn expand(
19 20
20fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 21fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
21 let mut input = TtCursor::new(input); 22 let mut input = TtCursor::new(input);
22 let bindings = match_lhs(&rule.lhs, &mut input)?; 23 let bindings = matcher::match_lhs(&rule.lhs, &mut input)?;
23 if !input.is_eof() { 24 if !input.is_eof() {
24 return Err(ExpandError::UnexpectedToken); 25 return Err(ExpandError::UnexpectedToken);
25 } 26 }
26 27 let res = transcriber::transcribe(&bindings, &rule.rhs)?;
27 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; 28 Ok(res)
28
29 expand_subtree(&rule.rhs, &mut ctx)
30} 29}
31 30
32/// The actual algorithm for expansion is not too hard, but is pretty tricky. 31/// The actual algorithm for expansion is not too hard, but is pretty tricky.
@@ -95,403 +94,6 @@ enum Fragment {
95 Ast(tt::TokenTree), 94 Ast(tt::TokenTree),
96} 95}
97 96
98impl Bindings {
99 fn push_optional(&mut self, name: &SmolStr) {
100 // FIXME: Do we have a better way to represent an empty token ?
101 // Insert an empty subtree for empty token
102 let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: vec![] }.into();
103 self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
104 }
105
106 fn push_empty(&mut self, name: &SmolStr) {
107 self.inner.insert(name.clone(), Binding::Empty);
108 }
109
110 fn contains(&self, name: &SmolStr) -> bool {
111 self.inner.contains_key(name)
112 }
113
114 fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
115 let mut b = self.inner.get(name).ok_or_else(|| {
116 ExpandError::BindingError(format!("could not find binding `{}`", name))
117 })?;
118 for &idx in nesting.iter() {
119 b = match b {
120 Binding::Fragment(_) => break,
121 Binding::Nested(bs) => bs.get(idx).ok_or_else(|| {
122 ExpandError::BindingError(format!("could not find nested binding `{}`", name))
123 })?,
124 Binding::Empty => {
125 return Err(ExpandError::BindingError(format!(
126 "could not find empty binding `{}`",
127 name
128 )))
129 }
130 };
131 }
132 match b {
133 Binding::Fragment(it) => Ok(it),
134 Binding::Nested(_) => Err(ExpandError::BindingError(format!(
135 "expected simple binding, found nested binding `{}`",
136 name
137 ))),
138 Binding::Empty => Err(ExpandError::BindingError(format!(
139 "expected simple binding, found empty binding `{}`",
140 name
141 ))),
142 }
143 }
144
145 fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
146 for (key, value) in nested.inner {
147 if !self.inner.contains_key(&key) {
148 self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
149 }
150 match self.inner.get_mut(&key) {
151 Some(Binding::Nested(it)) => {
152 // insert empty nested bindings before this one
153 while it.len() < idx {
154 it.push(Binding::Nested(vec![]));
155 }
156 it.push(value);
157 }
158 _ => {
159 return Err(ExpandError::BindingError(format!(
160 "could not find binding `{}`",
161 key
162 )));
163 }
164 }
165 }
166 Ok(())
167 }
168
169 fn merge(&mut self, nested: Bindings) {
170 self.inner.extend(nested.inner);
171 }
172}
173
174fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
175 let mut res = vec![];
176
177 for tkn in subtree.token_trees.iter() {
178 match tkn {
179 crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
180 res.push(text.clone());
181 }
182 crate::TokenTree::Subtree(subtree) => {
183 res.extend(collect_vars(subtree));
184 }
185 crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
186 res.extend(collect_vars(subtree));
187 }
188 _ => {}
189 }
190 }
191
192 res
193}
194
195fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
196 let mut res = Bindings::default();
197 for pat in pattern.token_trees.iter() {
198 match pat {
199 crate::TokenTree::Leaf(leaf) => match leaf {
200 crate::Leaf::Var(crate::Var { text, kind }) => {
201 let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
202 match match_meta_var(kind.as_str(), input)? {
203 Some(fragment) => {
204 res.inner.insert(text.clone(), Binding::Fragment(fragment));
205 }
206 None => res.push_optional(text),
207 }
208 }
209 crate::Leaf::Punct(punct) => {
210 if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
211 return Err(ExpandError::UnexpectedToken);
212 }
213 }
214 crate::Leaf::Ident(ident) => {
215 if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
216 return Err(ExpandError::UnexpectedToken);
217 }
218 }
219 crate::Leaf::Literal(literal) => {
220 if input.eat_literal().map(|i| &i.text) != Some(&literal.text) {
221 return Err(ExpandError::UnexpectedToken);
222 }
223 }
224 },
225 crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
226 // Dirty hack to make macro-expansion terminate.
227 // This should be replaced by a propper macro-by-example implementation
228 let mut limit = 65536;
229 let mut counter = 0;
230
231 let mut memento = input.save();
232
233 loop {
234 match match_lhs(subtree, input) {
235 Ok(nested) => {
236 limit -= 1;
237 if limit == 0 {
238 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
239 break;
240 }
241
242 memento = input.save();
243 res.push_nested(counter, nested)?;
244 counter += 1;
245 if counter == 1 {
246 if let crate::RepeatKind::ZeroOrOne = kind {
247 break;
248 }
249 }
250
251 if let Some(separator) = separator {
252 if !input
253 .eat_seperator()
254 .map(|sep| sep == *separator)
255 .unwrap_or(false)
256 {
257 input.rollback(memento);
258 break;
259 }
260 }
261 }
262 Err(_) => {
263 input.rollback(memento);
264 break;
265 }
266 }
267 }
268
269 match kind {
270 crate::RepeatKind::OneOrMore if counter == 0 => {
271 return Err(ExpandError::UnexpectedToken);
272 }
273 _ if counter == 0 => {
274 // Collect all empty variables in subtrees
275 collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
276 }
277 _ => {}
278 }
279 }
280 crate::TokenTree::Subtree(subtree) => {
281 let input_subtree =
282 input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
283 if subtree.delimiter != input_subtree.delimiter {
284 return Err(ExpandError::UnexpectedToken);
285 }
286
287 let mut input = TtCursor::new(input_subtree);
288 let bindings = match_lhs(&subtree, &mut input)?;
289 if !input.is_eof() {
290 return Err(ExpandError::UnexpectedToken);
291 }
292
293 res.merge(bindings);
294 }
295 }
296 }
297 Ok(res)
298}
299
300fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> {
301 let fragment = match kind {
302 "path" => Path,
303 "expr" => Expr,
304 "ty" => Type,
305 "pat" => Pattern,
306 "stmt" => Statement,
307 "block" => Block,
308 "meta" => MetaItem,
309 "item" => Item,
310 _ => {
311 let tt = match kind {
312 "ident" => {
313 let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
314 tt::Leaf::from(ident).into()
315 }
316 "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(),
317 "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(),
318 "literal" => {
319 let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
320 tt::Leaf::from(literal).into()
321 }
322 // `vis` is optional
323 "vis" => match input.try_eat_vis() {
324 Some(vis) => vis,
325 None => return Ok(None),
326 },
327 _ => return Err(ExpandError::UnexpectedToken),
328 };
329 return Ok(Some(Fragment::Tokens(tt)));
330 }
331 };
332 let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?;
333 let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
334 Ok(Some(fragment))
335}
336
337#[derive(Debug)]
338struct ExpandCtx<'a> {
339 bindings: &'a Bindings,
340 nesting: Vec<usize>,
341 var_expanded: bool,
342}
343
344fn expand_subtree(
345 template: &crate::Subtree,
346 ctx: &mut ExpandCtx,
347) -> Result<tt::Subtree, ExpandError> {
348 let mut buf: Vec<tt::TokenTree> = Vec::new();
349 for tt in template.token_trees.iter() {
350 let tt = expand_tt(tt, ctx)?;
351 push_fragment(&mut buf, tt);
352 }
353
354 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
355}
356
357fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> {
358 let res: tt::TokenTree = match template {
359 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
360 crate::TokenTree::Repeat(repeat) => {
361 let mut buf: Vec<tt::TokenTree> = Vec::new();
362 ctx.nesting.push(0);
363 // Dirty hack to make macro-expansion terminate.
364 // This should be replaced by a propper macro-by-example implementation
365 let mut limit = 65536;
366 let mut has_seps = 0;
367 let mut counter = 0;
368
369 // We store the old var expanded value, and restore it later
370 // It is because before this `$repeat`,
371 // it is possible some variables already expanad in the same subtree
372 //
373 // `some_var_expanded` keep check if the deeper subtree has expanded variables
374 let mut some_var_expanded = false;
375 let old_var_expanded = ctx.var_expanded;
376 ctx.var_expanded = false;
377
378 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
379 // if no var expanded in the child, we count it as a fail
380 if !ctx.var_expanded {
381 break;
382 }
383
384 // Reset `ctx.var_expandeded` to see if there is other expanded variable
385 // in the next matching
386 some_var_expanded = true;
387 ctx.var_expanded = false;
388
389 counter += 1;
390 limit -= 1;
391 if limit == 0 {
392 log::warn!(
393 "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
394 template,
395 ctx
396 );
397 break;
398 }
399
400 let idx = ctx.nesting.pop().unwrap();
401 ctx.nesting.push(idx + 1);
402 push_subtree(&mut buf, t);
403
404 if let Some(ref sep) = repeat.separator {
405 match sep {
406 crate::Separator::Ident(ident) => {
407 has_seps = 1;
408 buf.push(tt::Leaf::from(ident.clone()).into());
409 }
410 crate::Separator::Literal(lit) => {
411 has_seps = 1;
412 buf.push(tt::Leaf::from(lit.clone()).into());
413 }
414
415 crate::Separator::Puncts(puncts) => {
416 has_seps = puncts.len();
417 for punct in puncts {
418 buf.push(tt::Leaf::from(*punct).into());
419 }
420 }
421 }
422 }
423
424 if let crate::RepeatKind::ZeroOrOne = repeat.kind {
425 break;
426 }
427 }
428
429 // Restore the `var_expanded` by combining old one and the new one
430 ctx.var_expanded = some_var_expanded || old_var_expanded;
431
432 ctx.nesting.pop().unwrap();
433 for _ in 0..has_seps {
434 buf.pop();
435 }
436
437 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
438 return Err(ExpandError::UnexpectedToken);
439 }
440
441 // Check if it is a single token subtree without any delimiter
442 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
443 tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into()
444 }
445 crate::TokenTree::Leaf(leaf) => match leaf {
446 crate::Leaf::Ident(ident) => {
447 tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
448 .into()
449 }
450 crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
451 crate::Leaf::Var(v) => {
452 if v.text == "crate" {
453 // FIXME: Properly handle $crate token
454 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
455 .into()
456 } else if !ctx.bindings.contains(&v.text) {
457 // Note that it is possible to have a `$var` inside a macro which is not bound.
458 // For example:
459 // ```
460 // macro_rules! foo {
461 // ($a:ident, $b:ident, $c:tt) => {
462 // macro_rules! bar {
463 // ($bi:ident) => {
464 // fn $bi() -> u8 {$c}
465 // }
466 // }
467 // }
468 // ```
469 // We just treat it a normal tokens
470 tt::Subtree {
471 delimiter: tt::Delimiter::None,
472 token_trees: vec![
473 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
474 .into(),
475 tt::Leaf::from(tt::Ident {
476 text: v.text.clone(),
477 id: TokenId::unspecified(),
478 })
479 .into(),
480 ],
481 }
482 .into()
483 } else {
484 let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
485 ctx.var_expanded = true;
486 return Ok(fragment);
487 }
488 }
489 crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
490 },
491 };
492 Ok(Fragment::Tokens(res))
493}
494
495#[cfg(test)] 97#[cfg(test)]
496mod tests { 98mod tests {
497 use ra_syntax::{ast, AstNode}; 99 use ra_syntax::{ast, AstNode};
@@ -562,17 +164,3 @@ mod tests {
562 expand_rule(&rules.rules[0], &invocation_tt) 164 expand_rule(&rules.rules[0], &invocation_tt)
563 } 165 }
564} 166}
565
566fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
567 match fragment {
568 Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
569 Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt),
570 }
571}
572
573fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
574 match tt.delimiter {
575 tt::Delimiter::None => buf.extend(tt.token_trees),
576 _ => buf.push(tt.into()),
577 }
578}
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
new file mode 100644
index 000000000..100a3b0e0
--- /dev/null
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -0,0 +1,215 @@
1use crate::{
2 mbe_expander::{Binding, Bindings, Fragment},
3 tt_cursor::TtCursor,
4 ExpandError,
5};
6
7use ra_parser::FragmentKind::*;
8use ra_syntax::SmolStr;
9
10impl Bindings {
11 fn push_optional(&mut self, name: &SmolStr) {
12 // FIXME: Do we have a better way to represent an empty token ?
13 // Insert an empty subtree for empty token
14 let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: vec![] }.into();
15 self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
16 }
17
18 fn push_empty(&mut self, name: &SmolStr) {
19 self.inner.insert(name.clone(), Binding::Empty);
20 }
21
22 fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
23 for (key, value) in nested.inner {
24 if !self.inner.contains_key(&key) {
25 self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
26 }
27 match self.inner.get_mut(&key) {
28 Some(Binding::Nested(it)) => {
29 // insert empty nested bindings before this one
30 while it.len() < idx {
31 it.push(Binding::Nested(vec![]));
32 }
33 it.push(value);
34 }
35 _ => {
36 return Err(ExpandError::BindingError(format!(
37 "could not find binding `{}`",
38 key
39 )));
40 }
41 }
42 }
43 Ok(())
44 }
45
46 fn merge(&mut self, nested: Bindings) {
47 self.inner.extend(nested.inner);
48 }
49}
50
51pub(super) fn match_lhs(
52 pattern: &crate::Subtree,
53 input: &mut TtCursor,
54) -> Result<Bindings, ExpandError> {
55 let mut res = Bindings::default();
56 for pat in pattern.token_trees.iter() {
57 match pat {
58 crate::TokenTree::Leaf(leaf) => match leaf {
59 crate::Leaf::Var(crate::Var { text, kind }) => {
60 let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
61 match match_meta_var(kind.as_str(), input)? {
62 Some(fragment) => {
63 res.inner.insert(text.clone(), Binding::Fragment(fragment));
64 }
65 None => res.push_optional(text),
66 }
67 }
68 crate::Leaf::Punct(punct) => {
69 if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
70 return Err(ExpandError::UnexpectedToken);
71 }
72 }
73 crate::Leaf::Ident(ident) => {
74 if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
75 return Err(ExpandError::UnexpectedToken);
76 }
77 }
78 crate::Leaf::Literal(literal) => {
79 if input.eat_literal().map(|i| &i.text) != Some(&literal.text) {
80 return Err(ExpandError::UnexpectedToken);
81 }
82 }
83 },
84 crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
85 // Dirty hack to make macro-expansion terminate.
86 // This should be replaced by a propper macro-by-example implementation
87 let mut limit = 65536;
88 let mut counter = 0;
89
90 let mut memento = input.save();
91
92 loop {
93 match match_lhs(subtree, input) {
94 Ok(nested) => {
95 limit -= 1;
96 if limit == 0 {
97 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
98 break;
99 }
100
101 memento = input.save();
102 res.push_nested(counter, nested)?;
103 counter += 1;
104 if counter == 1 {
105 if let crate::RepeatKind::ZeroOrOne = kind {
106 break;
107 }
108 }
109
110 if let Some(separator) = separator {
111 if !input
112 .eat_seperator()
113 .map(|sep| sep == *separator)
114 .unwrap_or(false)
115 {
116 input.rollback(memento);
117 break;
118 }
119 }
120 }
121 Err(_) => {
122 input.rollback(memento);
123 break;
124 }
125 }
126 }
127
128 match kind {
129 crate::RepeatKind::OneOrMore if counter == 0 => {
130 return Err(ExpandError::UnexpectedToken);
131 }
132 _ if counter == 0 => {
133 // Collect all empty variables in subtrees
134 collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
135 }
136 _ => {}
137 }
138 }
139 crate::TokenTree::Subtree(subtree) => {
140 let input_subtree =
141 input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
142 if subtree.delimiter != input_subtree.delimiter {
143 return Err(ExpandError::UnexpectedToken);
144 }
145
146 let mut input = TtCursor::new(input_subtree);
147 let bindings = match_lhs(&subtree, &mut input)?;
148 if !input.is_eof() {
149 return Err(ExpandError::UnexpectedToken);
150 }
151
152 res.merge(bindings);
153 }
154 }
155 }
156 Ok(res)
157}
158
159fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> {
160 let fragment = match kind {
161 "path" => Path,
162 "expr" => Expr,
163 "ty" => Type,
164 "pat" => Pattern,
165 "stmt" => Statement,
166 "block" => Block,
167 "meta" => MetaItem,
168 "item" => Item,
169 _ => {
170 let tt = match kind {
171 "ident" => {
172 let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
173 tt::Leaf::from(ident).into()
174 }
175 "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(),
176 "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(),
177 "literal" => {
178 let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
179 tt::Leaf::from(literal).into()
180 }
181 // `vis` is optional
182 "vis" => match input.try_eat_vis() {
183 Some(vis) => vis,
184 None => return Ok(None),
185 },
186 _ => return Err(ExpandError::UnexpectedToken),
187 };
188 return Ok(Some(Fragment::Tokens(tt)));
189 }
190 };
191 let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?;
192 let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
193 Ok(Some(fragment))
194}
195
196fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
197 let mut res = Vec::new();
198
199 for tkn in subtree.token_trees.iter() {
200 match tkn {
201 crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
202 res.push(text.clone());
203 }
204 crate::TokenTree::Subtree(subtree) => {
205 res.extend(collect_vars(subtree));
206 }
207 crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
208 res.extend(collect_vars(subtree));
209 }
210 _ => {}
211 }
212 }
213
214 res
215}
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs
new file mode 100644
index 000000000..a3df1b7de
--- /dev/null
+++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs
@@ -0,0 +1,227 @@
1use ra_syntax::SmolStr;
2
3use crate::{
4 mbe_expander::{Binding, Bindings, Fragment},
5 ExpandError,
6};
7
8impl Bindings {
9 fn contains(&self, name: &SmolStr) -> bool {
10 self.inner.contains_key(name)
11 }
12
13 fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
14 let mut b = self.inner.get(name).ok_or_else(|| {
15 ExpandError::BindingError(format!("could not find binding `{}`", name))
16 })?;
17 for &idx in nesting.iter() {
18 b = match b {
19 Binding::Fragment(_) => break,
20 Binding::Nested(bs) => bs.get(idx).ok_or_else(|| {
21 ExpandError::BindingError(format!("could not find nested binding `{}`", name))
22 })?,
23 Binding::Empty => {
24 return Err(ExpandError::BindingError(format!(
25 "could not find empty binding `{}`",
26 name
27 )))
28 }
29 };
30 }
31 match b {
32 Binding::Fragment(it) => Ok(it),
33 Binding::Nested(_) => Err(ExpandError::BindingError(format!(
34 "expected simple binding, found nested binding `{}`",
35 name
36 ))),
37 Binding::Empty => Err(ExpandError::BindingError(format!(
38 "expected simple binding, found empty binding `{}`",
39 name
40 ))),
41 }
42 }
43}
44
45pub(super) fn transcribe(
46 bindings: &Bindings,
47 template: &crate::Subtree,
48) -> Result<tt::Subtree, ExpandError> {
49 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
50 expand_subtree(template, &mut ctx)
51}
52
53#[derive(Debug)]
54struct ExpandCtx<'a> {
55 bindings: &'a Bindings,
56 nesting: Vec<usize>,
57 var_expanded: bool,
58}
59
60fn expand_subtree(
61 template: &crate::Subtree,
62 ctx: &mut ExpandCtx,
63) -> Result<tt::Subtree, ExpandError> {
64 let mut buf: Vec<tt::TokenTree> = Vec::new();
65 for tt in template.token_trees.iter() {
66 let tt = expand_tt(tt, ctx)?;
67 push_fragment(&mut buf, tt);
68 }
69
70 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
71}
72
73fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> {
74 let res: tt::TokenTree = match template {
75 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
76 crate::TokenTree::Repeat(repeat) => {
77 let mut buf: Vec<tt::TokenTree> = Vec::new();
78 ctx.nesting.push(0);
79 // Dirty hack to make macro-expansion terminate.
80 // This should be replaced by a propper macro-by-example implementation
81 let mut limit = 65536;
82 let mut has_seps = 0;
83 let mut counter = 0;
84
85 // We store the old var expanded value, and restore it later
86 // It is because before this `$repeat`,
87 // it is possible some variables already expanad in the same subtree
88 //
89 // `some_var_expanded` keep check if the deeper subtree has expanded variables
90 let mut some_var_expanded = false;
91 let old_var_expanded = ctx.var_expanded;
92 ctx.var_expanded = false;
93
94 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
95 // if no var expanded in the child, we count it as a fail
96 if !ctx.var_expanded {
97 break;
98 }
99
100 // Reset `ctx.var_expandeded` to see if there is other expanded variable
101 // in the next matching
102 some_var_expanded = true;
103 ctx.var_expanded = false;
104
105 counter += 1;
106 limit -= 1;
107 if limit == 0 {
108 log::warn!(
109 "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
110 template,
111 ctx
112 );
113 break;
114 }
115
116 let idx = ctx.nesting.pop().unwrap();
117 ctx.nesting.push(idx + 1);
118 push_subtree(&mut buf, t);
119
120 if let Some(ref sep) = repeat.separator {
121 match sep {
122 crate::Separator::Ident(ident) => {
123 has_seps = 1;
124 buf.push(tt::Leaf::from(ident.clone()).into());
125 }
126 crate::Separator::Literal(lit) => {
127 has_seps = 1;
128 buf.push(tt::Leaf::from(lit.clone()).into());
129 }
130
131 crate::Separator::Puncts(puncts) => {
132 has_seps = puncts.len();
133 for punct in puncts {
134 buf.push(tt::Leaf::from(*punct).into());
135 }
136 }
137 }
138 }
139
140 if let crate::RepeatKind::ZeroOrOne = repeat.kind {
141 break;
142 }
143 }
144
145 // Restore the `var_expanded` by combining old one and the new one
146 ctx.var_expanded = some_var_expanded || old_var_expanded;
147
148 ctx.nesting.pop().unwrap();
149 for _ in 0..has_seps {
150 buf.pop();
151 }
152
153 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
154 return Err(ExpandError::UnexpectedToken);
155 }
156
157 // Check if it is a single token subtree without any delimiter
158 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
159 tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into()
160 }
161 crate::TokenTree::Leaf(leaf) => match leaf {
162 crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident {
163 text: ident.text.clone(),
164 id: tt::TokenId::unspecified(),
165 })
166 .into(),
167 crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
168 crate::Leaf::Var(v) => {
169 if v.text == "crate" {
170 // FIXME: Properly handle $crate token
171 tt::Leaf::from(tt::Ident {
172 text: "$crate".into(),
173 id: tt::TokenId::unspecified(),
174 })
175 .into()
176 } else if !ctx.bindings.contains(&v.text) {
177 // Note that it is possible to have a `$var` inside a macro which is not bound.
178 // For example:
179 // ```
180 // macro_rules! foo {
181 // ($a:ident, $b:ident, $c:tt) => {
182 // macro_rules! bar {
183 // ($bi:ident) => {
184 // fn $bi() -> u8 {$c}
185 // }
186 // }
187 // }
188 // ```
189 // We just treat it a normal tokens
190 tt::Subtree {
191 delimiter: tt::Delimiter::None,
192 token_trees: vec![
193 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
194 .into(),
195 tt::Leaf::from(tt::Ident {
196 text: v.text.clone(),
197 id: tt::TokenId::unspecified(),
198 })
199 .into(),
200 ],
201 }
202 .into()
203 } else {
204 let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
205 ctx.var_expanded = true;
206 return Ok(fragment);
207 }
208 }
209 crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
210 },
211 };
212 Ok(Fragment::Tokens(res))
213}
214
215fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
216 match fragment {
217 Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
218 Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt),
219 }
220}
221
222fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
223 match tt.delimiter {
224 tt::Delimiter::None => buf.extend(tt.token_trees),
225 _ => buf.push(tt.into()),
226 }
227}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-09 13:00:01 +0000