aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe/src/mbe_expander.rs
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2019-09-17 00:06:38 +0100
committerGitHub <[email protected]>2019-09-17 00:06:38 +0100
commit8eb2697b7d2a98c952b3acd1711829a13e13cab1 (patch)
treeb8bee4dd75c07f55b9f0d7e86380aba904557c3a /crates/ra_mbe/src/mbe_expander.rs
parentba583091e60553633dd3cc9ab37a1d9f64827a1e (diff)
parent37ef8927c373b8eadd63edc1f70055428c49290e (diff)
Merge #1855
1855: split mbe expander code into two modules r=matklad a=matklad Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_mbe/src/mbe_expander.rs')
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs424
1 files changed, 6 insertions, 418 deletions
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 78df96880..15d9d83e2 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -2,10 +2,11 @@
2//! `tt::TokenTree` representing an argument of macro invocation, and produces a 2//! `tt::TokenTree` representing an argument of macro invocation, and produces a
3//! `tt::TokenTree` for the result of the expansion. 3//! `tt::TokenTree` for the result of the expansion.
4 4
5use ra_parser::FragmentKind::*; 5mod matcher;
6mod transcriber;
7
6use ra_syntax::SmolStr; 8use ra_syntax::SmolStr;
7use rustc_hash::FxHashMap; 9use rustc_hash::FxHashMap;
8use tt::TokenId;
9 10
10use crate::tt_cursor::TtCursor; 11use crate::tt_cursor::TtCursor;
11use crate::ExpandError; 12use crate::ExpandError;
@@ -19,14 +20,12 @@ pub(crate) fn expand(
19 20
20fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 21fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
21 let mut input = TtCursor::new(input); 22 let mut input = TtCursor::new(input);
22 let bindings = match_lhs(&rule.lhs, &mut input)?; 23 let bindings = matcher::match_lhs(&rule.lhs, &mut input)?;
23 if !input.is_eof() { 24 if !input.is_eof() {
24 return Err(ExpandError::UnexpectedToken); 25 return Err(ExpandError::UnexpectedToken);
25 } 26 }
26 27 let res = transcriber::transcribe(&bindings, &rule.rhs)?;
27 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; 28 Ok(res)
28
29 expand_subtree(&rule.rhs, &mut ctx)
30} 29}
31 30
32/// The actual algorithm for expansion is not too hard, but is pretty tricky. 31/// The actual algorithm for expansion is not too hard, but is pretty tricky.
@@ -95,403 +94,6 @@ enum Fragment {
95 Ast(tt::TokenTree), 94 Ast(tt::TokenTree),
96} 95}
97 96
98impl Bindings {
99 fn push_optional(&mut self, name: &SmolStr) {
100 // FIXME: Do we have a better way to represent an empty token ?
101 // Insert an empty subtree for empty token
102 let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: vec![] }.into();
103 self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
104 }
105
106 fn push_empty(&mut self, name: &SmolStr) {
107 self.inner.insert(name.clone(), Binding::Empty);
108 }
109
110 fn contains(&self, name: &SmolStr) -> bool {
111 self.inner.contains_key(name)
112 }
113
114 fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
115 let mut b = self.inner.get(name).ok_or_else(|| {
116 ExpandError::BindingError(format!("could not find binding `{}`", name))
117 })?;
118 for &idx in nesting.iter() {
119 b = match b {
120 Binding::Fragment(_) => break,
121 Binding::Nested(bs) => bs.get(idx).ok_or_else(|| {
122 ExpandError::BindingError(format!("could not find nested binding `{}`", name))
123 })?,
124 Binding::Empty => {
125 return Err(ExpandError::BindingError(format!(
126 "could not find empty binding `{}`",
127 name
128 )))
129 }
130 };
131 }
132 match b {
133 Binding::Fragment(it) => Ok(it),
134 Binding::Nested(_) => Err(ExpandError::BindingError(format!(
135 "expected simple binding, found nested binding `{}`",
136 name
137 ))),
138 Binding::Empty => Err(ExpandError::BindingError(format!(
139 "expected simple binding, found empty binding `{}`",
140 name
141 ))),
142 }
143 }
144
145 fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
146 for (key, value) in nested.inner {
147 if !self.inner.contains_key(&key) {
148 self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
149 }
150 match self.inner.get_mut(&key) {
151 Some(Binding::Nested(it)) => {
152 // insert empty nested bindings before this one
153 while it.len() < idx {
154 it.push(Binding::Nested(vec![]));
155 }
156 it.push(value);
157 }
158 _ => {
159 return Err(ExpandError::BindingError(format!(
160 "could not find binding `{}`",
161 key
162 )));
163 }
164 }
165 }
166 Ok(())
167 }
168
169 fn merge(&mut self, nested: Bindings) {
170 self.inner.extend(nested.inner);
171 }
172}
173
174fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> {
175 let mut res = vec![];
176
177 for tkn in subtree.token_trees.iter() {
178 match tkn {
179 crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => {
180 res.push(text.clone());
181 }
182 crate::TokenTree::Subtree(subtree) => {
183 res.extend(collect_vars(subtree));
184 }
185 crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
186 res.extend(collect_vars(subtree));
187 }
188 _ => {}
189 }
190 }
191
192 res
193}
194
195fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings, ExpandError> {
196 let mut res = Bindings::default();
197 for pat in pattern.token_trees.iter() {
198 match pat {
199 crate::TokenTree::Leaf(leaf) => match leaf {
200 crate::Leaf::Var(crate::Var { text, kind }) => {
201 let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
202 match match_meta_var(kind.as_str(), input)? {
203 Some(fragment) => {
204 res.inner.insert(text.clone(), Binding::Fragment(fragment));
205 }
206 None => res.push_optional(text),
207 }
208 }
209 crate::Leaf::Punct(punct) => {
210 if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) {
211 return Err(ExpandError::UnexpectedToken);
212 }
213 }
214 crate::Leaf::Ident(ident) => {
215 if input.eat_ident().map(|i| &i.text) != Some(&ident.text) {
216 return Err(ExpandError::UnexpectedToken);
217 }
218 }
219 crate::Leaf::Literal(literal) => {
220 if input.eat_literal().map(|i| &i.text) != Some(&literal.text) {
221 return Err(ExpandError::UnexpectedToken);
222 }
223 }
224 },
225 crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => {
226 // Dirty hack to make macro-expansion terminate.
227 // This should be replaced by a propper macro-by-example implementation
228 let mut limit = 65536;
229 let mut counter = 0;
230
231 let mut memento = input.save();
232
233 loop {
234 match match_lhs(subtree, input) {
235 Ok(nested) => {
236 limit -= 1;
237 if limit == 0 {
238 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator);
239 break;
240 }
241
242 memento = input.save();
243 res.push_nested(counter, nested)?;
244 counter += 1;
245 if counter == 1 {
246 if let crate::RepeatKind::ZeroOrOne = kind {
247 break;
248 }
249 }
250
251 if let Some(separator) = separator {
252 if !input
253 .eat_seperator()
254 .map(|sep| sep == *separator)
255 .unwrap_or(false)
256 {
257 input.rollback(memento);
258 break;
259 }
260 }
261 }
262 Err(_) => {
263 input.rollback(memento);
264 break;
265 }
266 }
267 }
268
269 match kind {
270 crate::RepeatKind::OneOrMore if counter == 0 => {
271 return Err(ExpandError::UnexpectedToken);
272 }
273 _ if counter == 0 => {
274 // Collect all empty variables in subtrees
275 collect_vars(subtree).iter().for_each(|s| res.push_empty(s));
276 }
277 _ => {}
278 }
279 }
280 crate::TokenTree::Subtree(subtree) => {
281 let input_subtree =
282 input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?;
283 if subtree.delimiter != input_subtree.delimiter {
284 return Err(ExpandError::UnexpectedToken);
285 }
286
287 let mut input = TtCursor::new(input_subtree);
288 let bindings = match_lhs(&subtree, &mut input)?;
289 if !input.is_eof() {
290 return Err(ExpandError::UnexpectedToken);
291 }
292
293 res.merge(bindings);
294 }
295 }
296 }
297 Ok(res)
298}
299
300fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> {
301 let fragment = match kind {
302 "path" => Path,
303 "expr" => Expr,
304 "ty" => Type,
305 "pat" => Pattern,
306 "stmt" => Statement,
307 "block" => Block,
308 "meta" => MetaItem,
309 "item" => Item,
310 _ => {
311 let tt = match kind {
312 "ident" => {
313 let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone();
314 tt::Leaf::from(ident).into()
315 }
316 "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(),
317 "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(),
318 "literal" => {
319 let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone();
320 tt::Leaf::from(literal).into()
321 }
322 // `vis` is optional
323 "vis" => match input.try_eat_vis() {
324 Some(vis) => vis,
325 None => return Ok(None),
326 },
327 _ => return Err(ExpandError::UnexpectedToken),
328 };
329 return Ok(Some(Fragment::Tokens(tt)));
330 }
331 };
332 let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?;
333 let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
334 Ok(Some(fragment))
335}
336
337#[derive(Debug)]
338struct ExpandCtx<'a> {
339 bindings: &'a Bindings,
340 nesting: Vec<usize>,
341 var_expanded: bool,
342}
343
344fn expand_subtree(
345 template: &crate::Subtree,
346 ctx: &mut ExpandCtx,
347) -> Result<tt::Subtree, ExpandError> {
348 let mut buf: Vec<tt::TokenTree> = Vec::new();
349 for tt in template.token_trees.iter() {
350 let tt = expand_tt(tt, ctx)?;
351 push_fragment(&mut buf, tt);
352 }
353
354 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
355}
356
357fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> {
358 let res: tt::TokenTree = match template {
359 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
360 crate::TokenTree::Repeat(repeat) => {
361 let mut buf: Vec<tt::TokenTree> = Vec::new();
362 ctx.nesting.push(0);
363 // Dirty hack to make macro-expansion terminate.
364 // This should be replaced by a propper macro-by-example implementation
365 let mut limit = 65536;
366 let mut has_seps = 0;
367 let mut counter = 0;
368
369 // We store the old var expanded value, and restore it later
370 // It is because before this `$repeat`,
371 // it is possible some variables already expanad in the same subtree
372 //
373 // `some_var_expanded` keep check if the deeper subtree has expanded variables
374 let mut some_var_expanded = false;
375 let old_var_expanded = ctx.var_expanded;
376 ctx.var_expanded = false;
377
378 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) {
379 // if no var expanded in the child, we count it as a fail
380 if !ctx.var_expanded {
381 break;
382 }
383
384 // Reset `ctx.var_expandeded` to see if there is other expanded variable
385 // in the next matching
386 some_var_expanded = true;
387 ctx.var_expanded = false;
388
389 counter += 1;
390 limit -= 1;
391 if limit == 0 {
392 log::warn!(
393 "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
394 template,
395 ctx
396 );
397 break;
398 }
399
400 let idx = ctx.nesting.pop().unwrap();
401 ctx.nesting.push(idx + 1);
402 push_subtree(&mut buf, t);
403
404 if let Some(ref sep) = repeat.separator {
405 match sep {
406 crate::Separator::Ident(ident) => {
407 has_seps = 1;
408 buf.push(tt::Leaf::from(ident.clone()).into());
409 }
410 crate::Separator::Literal(lit) => {
411 has_seps = 1;
412 buf.push(tt::Leaf::from(lit.clone()).into());
413 }
414
415 crate::Separator::Puncts(puncts) => {
416 has_seps = puncts.len();
417 for punct in puncts {
418 buf.push(tt::Leaf::from(*punct).into());
419 }
420 }
421 }
422 }
423
424 if let crate::RepeatKind::ZeroOrOne = repeat.kind {
425 break;
426 }
427 }
428
429 // Restore the `var_expanded` by combining old one and the new one
430 ctx.var_expanded = some_var_expanded || old_var_expanded;
431
432 ctx.nesting.pop().unwrap();
433 for _ in 0..has_seps {
434 buf.pop();
435 }
436
437 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
438 return Err(ExpandError::UnexpectedToken);
439 }
440
441 // Check if it is a single token subtree without any delimiter
442 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
443 tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into()
444 }
445 crate::TokenTree::Leaf(leaf) => match leaf {
446 crate::Leaf::Ident(ident) => {
447 tt::Leaf::from(tt::Ident { text: ident.text.clone(), id: TokenId::unspecified() })
448 .into()
449 }
450 crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
451 crate::Leaf::Var(v) => {
452 if v.text == "crate" {
453 // FIXME: Properly handle $crate token
454 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: TokenId::unspecified() })
455 .into()
456 } else if !ctx.bindings.contains(&v.text) {
457 // Note that it is possible to have a `$var` inside a macro which is not bound.
458 // For example:
459 // ```
460 // macro_rules! foo {
461 // ($a:ident, $b:ident, $c:tt) => {
462 // macro_rules! bar {
463 // ($bi:ident) => {
464 // fn $bi() -> u8 {$c}
465 // }
466 // }
467 // }
468 // ```
469 // We just treat it a normal tokens
470 tt::Subtree {
471 delimiter: tt::Delimiter::None,
472 token_trees: vec![
473 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
474 .into(),
475 tt::Leaf::from(tt::Ident {
476 text: v.text.clone(),
477 id: TokenId::unspecified(),
478 })
479 .into(),
480 ],
481 }
482 .into()
483 } else {
484 let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
485 ctx.var_expanded = true;
486 return Ok(fragment);
487 }
488 }
489 crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
490 },
491 };
492 Ok(Fragment::Tokens(res))
493}
494
495#[cfg(test)] 97#[cfg(test)]
496mod tests { 98mod tests {
497 use ra_syntax::{ast, AstNode}; 99 use ra_syntax::{ast, AstNode};
@@ -562,17 +164,3 @@ mod tests {
562 expand_rule(&rules.rules[0], &invocation_tt) 164 expand_rule(&rules.rules[0], &invocation_tt)
563 } 165 }
564} 166}
565
566fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
567 match fragment {
568 Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
569 Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt),
570 }
571}
572
573fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
574 match tt.delimiter {
575 tt::Delimiter::None => buf.extend(tt.token_trees),
576 _ => buf.push(tt.into()),
577 }
578}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-07 03:54:31 +0000