From 4551182f94fe81c314f79ddf8916a5520cfd03b0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 17 Sep 2019 02:54:22 +0300 Subject: use usual token tree for macro expansion --- crates/ra_mbe/src/mbe_expander/transcriber.rs | 288 +++++++++++++------------- 1 file changed, 146 insertions(+), 142 deletions(-) (limited to 'crates/ra_mbe/src/mbe_expander/transcriber.rs') diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index a3df1b7de..c22680b93 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs @@ -1,16 +1,20 @@ +//! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like +//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` + use ra_syntax::SmolStr; use crate::{ mbe_expander::{Binding, Bindings, Fragment}, + parser::{parse_template, Op, RepeatKind, Separator}, ExpandError, }; impl Bindings { - fn contains(&self, name: &SmolStr) -> bool { + fn contains(&self, name: &str) -> bool { self.inner.contains_key(name) } - fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { + fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> { let mut b = self.inner.get(name).ok_or_else(|| { ExpandError::BindingError(format!("could not find binding `{}`", name)) })?; @@ -43,11 +47,12 @@ impl Bindings { } pub(super) fn transcribe( + template: &tt::Subtree, bindings: &Bindings, - template: &crate::Subtree, ) -> Result { + assert!(template.delimiter == tt::Delimiter::None); let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; - expand_subtree(template, &mut ctx) + expand_subtree(&mut ctx, template) } #[derive(Debug)] @@ -57,159 +62,158 @@ struct ExpandCtx<'a> { var_expanded: bool, } -fn expand_subtree( - template: &crate::Subtree, - ctx: &mut ExpandCtx, -) -> Result { +fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result { let mut buf: Vec = Vec::new(); - for tt in template.token_trees.iter() { - let tt = expand_tt(tt, ctx)?; - push_fragment(&mut buf, tt); + for op in parse_template(template) { + match op? { + Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), + Op::TokenTree(tt::TokenTree::Subtree(tt)) => { + let tt = expand_subtree(ctx, tt)?; + buf.push(tt.into()); + } + Op::Var { name, kind: _ } => { + let fragment = expand_var(ctx, name)?; + push_fragment(&mut buf, fragment); + } + Op::Repeat { subtree, kind, separator } => { + let fragment = expand_repeat(ctx, subtree, kind, separator)?; + push_fragment(&mut buf, fragment) + } + } } - Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) } -fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result { - let res: tt::TokenTree = match template { - crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), - crate::TokenTree::Repeat(repeat) => { - let mut buf: Vec = Vec::new(); - ctx.nesting.push(0); - // Dirty hack to make macro-expansion terminate. - // This should be replaced by a propper macro-by-example implementation - let mut limit = 65536; - let mut has_seps = 0; - let mut counter = 0; - - // We store the old var expanded value, and restore it later - // It is because before this `$repeat`, - // it is possible some variables already expanad in the same subtree - // - // `some_var_expanded` keep check if the deeper subtree has expanded variables - let mut some_var_expanded = false; - let old_var_expanded = ctx.var_expanded; - ctx.var_expanded = false; - - while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { - // if no var expanded in the child, we count it as a fail - if !ctx.var_expanded { - break; - } +fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result { + let res = if v == "crate" { + // FIXME: Properly handle $crate token + let tt = + tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) + .into(); + Fragment::Tokens(tt) + } else if !ctx.bindings.contains(v) { + // Note that it is possible to have a `$var` inside a macro which is not bound. + // For example: + // ``` + // macro_rules! foo { + // ($a:ident, $b:ident, $c:tt) => { + // macro_rules! bar { + // ($bi:ident) => { + // fn $bi() -> u8 {$c} + // } + // } + // } + // ``` + // We just treat it a normal tokens + let tt = tt::Subtree { + delimiter: tt::Delimiter::None, + token_trees: vec![ + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), + tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) + .into(), + ], + } + .into(); + Fragment::Tokens(tt) + } else { + let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone(); + ctx.var_expanded = true; + fragment + }; + Ok(res) +} - // Reset `ctx.var_expandeded` to see if there is other expanded variable - // in the next matching - some_var_expanded = true; - ctx.var_expanded = false; - - counter += 1; - limit -= 1; - if limit == 0 { - log::warn!( - "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", - template, - ctx - ); - break; - } +fn expand_repeat( + ctx: &mut ExpandCtx, + template: &tt::Subtree, + kind: RepeatKind, + separator: Option, +) -> Result { + let mut buf: Vec = Vec::new(); + ctx.nesting.push(0); + // Dirty hack to make macro-expansion terminate. + // This should be replaced by a propper macro-by-example implementation + let mut limit = 65536; + let mut has_seps = 0; + let mut counter = 0; + + // We store the old var expanded value, and restore it later + // It is because before this `$repeat`, + // it is possible some variables already expanad in the same subtree + // + // `some_var_expanded` keep check if the deeper subtree has expanded variables + let mut some_var_expanded = false; + let old_var_expanded = ctx.var_expanded; + ctx.var_expanded = false; + + while let Ok(mut t) = expand_subtree(ctx, template) { + t.delimiter = tt::Delimiter::None; + // if no var expanded in the child, we count it as a fail + if !ctx.var_expanded { + break; + } - let idx = ctx.nesting.pop().unwrap(); - ctx.nesting.push(idx + 1); - push_subtree(&mut buf, t); - - if let Some(ref sep) = repeat.separator { - match sep { - crate::Separator::Ident(ident) => { - has_seps = 1; - buf.push(tt::Leaf::from(ident.clone()).into()); - } - crate::Separator::Literal(lit) => { - has_seps = 1; - buf.push(tt::Leaf::from(lit.clone()).into()); - } - - crate::Separator::Puncts(puncts) => { - has_seps = puncts.len(); - for punct in puncts { - buf.push(tt::Leaf::from(*punct).into()); - } - } - } + // Reset `ctx.var_expandeded` to see if there is other expanded variable + // in the next matching + some_var_expanded = true; + ctx.var_expanded = false; + + counter += 1; + limit -= 1; + if limit == 0 { + log::warn!( + "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", + template, + ctx + ); + break; + } + + let idx = ctx.nesting.pop().unwrap(); + ctx.nesting.push(idx + 1); + push_subtree(&mut buf, t); + + if let Some(ref sep) = separator { + match sep { + Separator::Ident(ident) => { + has_seps = 1; + buf.push(tt::Leaf::from(ident.clone()).into()); + } + Separator::Literal(lit) => { + has_seps = 1; + buf.push(tt::Leaf::from(lit.clone()).into()); } - if let crate::RepeatKind::ZeroOrOne = repeat.kind { - break; + Separator::Puncts(puncts) => { + has_seps = puncts.len(); + for punct in puncts { + buf.push(tt::Leaf::from(*punct).into()); + } } } + } - // Restore the `var_expanded` by combining old one and the new one - ctx.var_expanded = some_var_expanded || old_var_expanded; + if RepeatKind::ZeroOrOne == kind { + break; + } + } - ctx.nesting.pop().unwrap(); - for _ in 0..has_seps { - buf.pop(); - } + // Restore the `var_expanded` by combining old one and the new one + ctx.var_expanded = some_var_expanded || old_var_expanded; - if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { - return Err(ExpandError::UnexpectedToken); - } + ctx.nesting.pop().unwrap(); + for _ in 0..has_seps { + buf.pop(); + } - // Check if it is a single token subtree without any delimiter - // e.g {Delimiter:None> ['>'] /Delimiter:None>} - tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() - } - crate::TokenTree::Leaf(leaf) => match leaf { - crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident { - text: ident.text.clone(), - id: tt::TokenId::unspecified(), - }) - .into(), - crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(), - crate::Leaf::Var(v) => { - if v.text == "crate" { - // FIXME: Properly handle $crate token - tt::Leaf::from(tt::Ident { - text: "$crate".into(), - id: tt::TokenId::unspecified(), - }) - .into() - } else if !ctx.bindings.contains(&v.text) { - // Note that it is possible to have a `$var` inside a macro which is not bound. - // For example: - // ``` - // macro_rules! foo { - // ($a:ident, $b:ident, $c:tt) => { - // macro_rules! bar { - // ($bi:ident) => { - // fn $bi() -> u8 {$c} - // } - // } - // } - // ``` - // We just treat it a normal tokens - tt::Subtree { - delimiter: tt::Delimiter::None, - token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }) - .into(), - tt::Leaf::from(tt::Ident { - text: v.text.clone(), - id: tt::TokenId::unspecified(), - }) - .into(), - ], - } - .into() - } else { - let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); - ctx.var_expanded = true; - return Ok(fragment); - } - } - crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(), - }, - }; - Ok(Fragment::Tokens(res)) + if RepeatKind::OneOrMore == kind && counter == 0 { + return Err(ExpandError::UnexpectedToken); + } + + // Check if it is a single token subtree without any delimiter + // e.g {Delimiter:None> ['>'] /Delimiter:None>} + let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into(); + Ok(Fragment::Tokens(tt)) } fn push_fragment(buf: &mut Vec, fragment: Fragment) { -- cgit v1.2.3