From 04af290d4e08e282dda0c0273f20a46f381224f4 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Mon, 4 Nov 2019 23:22:18 +0800 Subject: Add TokenId Shif in macro_rules --- crates/ra_mbe/src/lib.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++-- crates/ra_mbe/src/tests.rs | 27 +++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index afdbee84e..7d4a5f307 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -42,6 +42,8 @@ pub use crate::syntax_bridge::{ #[derive(Clone, Debug, PartialEq, Eq)] pub struct MacroRules { pub(crate) rules: Vec, + /// Highest id of the token we have in TokenMap + pub(crate) shift: Option, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -50,6 +52,42 @@ pub(crate) struct Rule { pub(crate) rhs: tt::Subtree, } +/// Find the "shift" (the highest id of the TokenId) inside a subtree +fn find_subtree_shift(tt: &tt::Subtree, mut cur: Option) -> Option { + use std::cmp::max; + + for t in &tt.token_trees { + cur = match t { + tt::TokenTree::Leaf(leaf) => match leaf { + tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => { + Some(max(cur.unwrap_or(0), ident.id.0)) + } + _ => cur, + }, + tt::TokenTree::Subtree(tt) => find_subtree_shift(tt, cur), + } + } + + cur +} + +/// Shift given TokenTree token id +fn shift_token_tree(tt: &mut tt::Subtree, shift: u32) { + for t in tt.token_trees.iter_mut() { + match t { + tt::TokenTree::Leaf(leaf) => match leaf { + tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => { + // Note that TokenId is started from zero, + // We have to add 1 to prevent duplication. + ident.id.0 += shift + 1; + } + _ => (), + }, + tt::TokenTree::Subtree(tt) => shift_token_tree(tt, shift), + } + } +} + impl MacroRules { pub fn parse(tt: &tt::Subtree) -> Result { // Note: this parsing can be implemented using mbe machinery itself, by @@ -72,10 +110,17 @@ impl MacroRules { validate(&rule.lhs)?; } - Ok(MacroRules { rules }) + Ok(MacroRules { rules, shift: find_subtree_shift(tt, None) }) } + pub fn expand(&self, tt: &tt::Subtree) -> Result { - mbe_expander::expand(self, tt) + // apply shift + let mut tt = tt.clone(); + if let Some(shift) = self.shift { + shift_token_tree(&mut tt, shift) + } + + mbe_expander::expand(self, &tt) } } diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index f34ab52e1..a23e3afe3 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs @@ -58,6 +58,33 @@ mod rule_parsing { // * Port the test to rust and add it to this module // * Make it pass :-) +#[test] +fn test_token_id_shift() { + let macro_definition = r#" +macro_rules! foobar { + ($e:ident) => { foo bar $e } +} +"#; + let rules = create_rules(macro_definition); + let expansion = expand(&rules, "foobar!(baz);"); + + fn get_id(t: &tt::TokenTree) -> Option { + if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = t { + return Some(ident.id.0); + } + None + } + + assert_eq!(expansion.token_trees.len(), 3); + // ($e:ident) => { foo bar $e } + // 0 1 2 3 4 + assert_eq!(get_id(&expansion.token_trees[0]), Some(2)); + assert_eq!(get_id(&expansion.token_trees[1]), Some(3)); + + // So baz should be 5 + assert_eq!(get_id(&expansion.token_trees[2]), Some(5)); +} + #[test] fn test_convert_tt() { let macro_definition = r#" -- cgit v1.2.3 From 63e42bb5bdd57b8b0155d9e38ebe78e0e864c4d5 Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 00:16:03 +0800 Subject: Change to better naming --- crates/ra_mbe/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 7d4a5f307..f51149258 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -72,7 +72,7 @@ fn find_subtree_shift(tt: &tt::Subtree, mut cur: Option) -> Option { } /// Shift given TokenTree token id -fn shift_token_tree(tt: &mut tt::Subtree, shift: u32) { +fn shift_subtree(tt: &mut tt::Subtree, shift: u32) { for t in tt.token_trees.iter_mut() { match t { tt::TokenTree::Leaf(leaf) => match leaf { @@ -83,7 +83,7 @@ fn shift_token_tree(tt: &mut tt::Subtree, shift: u32) { } _ => (), }, - tt::TokenTree::Subtree(tt) => shift_token_tree(tt, shift), + tt::TokenTree::Subtree(tt) => shift_subtree(tt, shift), } } } @@ -117,7 +117,7 @@ impl MacroRules { // apply shift let mut tt = tt.clone(); if let Some(shift) = self.shift { - shift_token_tree(&mut tt, shift) + shift_subtree(&mut tt, shift) } mbe_expander::expand(self, &tt) -- cgit v1.2.3 From 42661a3b27f039f4eb11ef7c5f4733bb79b4a1ec Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 00:33:19 +0800 Subject: Change Option to u32 for shift value --- crates/ra_mbe/src/lib.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index f51149258..1c076b36b 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -43,7 +43,7 @@ pub use crate::syntax_bridge::{ pub struct MacroRules { pub(crate) rules: Vec, /// Highest id of the token we have in TokenMap - pub(crate) shift: Option, + pub(crate) shift: u32, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -53,14 +53,14 @@ pub(crate) struct Rule { } /// Find the "shift" (the highest id of the TokenId) inside a subtree -fn find_subtree_shift(tt: &tt::Subtree, mut cur: Option) -> Option { +fn find_subtree_shift(tt: &tt::Subtree, mut cur: u32) -> u32 { use std::cmp::max; for t in &tt.token_trees { cur = match t { tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => { - Some(max(cur.unwrap_or(0), ident.id.0)) + max(cur, ident.id.0) } _ => cur, }, @@ -110,16 +110,13 @@ impl MacroRules { validate(&rule.lhs)?; } - Ok(MacroRules { rules, shift: find_subtree_shift(tt, None) }) + Ok(MacroRules { rules, shift: find_subtree_shift(tt, 0) }) } pub fn expand(&self, tt: &tt::Subtree) -> Result { // apply shift let mut tt = tt.clone(); - if let Some(shift) = self.shift { - shift_subtree(&mut tt, shift) - } - + shift_subtree(&mut tt, self.shift); mbe_expander::expand(self, &tt) } } -- cgit v1.2.3 From 188a1412b9d21ead5d823ce3ca00d3b4f823221b Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 01:01:05 +0800 Subject: Refactor and rename --- crates/ra_mbe/src/lib.rs | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index 1c076b36b..b92312d52 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -52,23 +52,21 @@ pub(crate) struct Rule { pub(crate) rhs: tt::Subtree, } -/// Find the "shift" (the highest id of the TokenId) inside a subtree -fn find_subtree_shift(tt: &tt::Subtree, mut cur: u32) -> u32 { - use std::cmp::max; - - for t in &tt.token_trees { - cur = match t { - tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => { - max(cur, ident.id.0) - } - _ => cur, - }, - tt::TokenTree::Subtree(tt) => find_subtree_shift(tt, cur), - } - } - - cur +// Find the max token id inside a subtree +fn max_id(subtree: &tt::Subtree) -> Option { + subtree + .token_trees + .iter() + .filter_map(|tt| match tt { + tt::TokenTree::Subtree(subtree) => max_id(subtree), + tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) + if ident.id != tt::TokenId::unspecified() => + { + Some(ident.id.0) + } + _ => None, + }) + .max() } /// Shift given TokenTree token id @@ -77,9 +75,7 @@ fn shift_subtree(tt: &mut tt::Subtree, shift: u32) { match t { tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => { - // Note that TokenId is started from zero, - // We have to add 1 to prevent duplication. - ident.id.0 += shift + 1; + ident.id.0 += shift; } _ => (), }, @@ -110,7 +106,10 @@ impl MacroRules { validate(&rule.lhs)?; } - Ok(MacroRules { rules, shift: find_subtree_shift(tt, 0) }) + // Note that TokenId is started from zero, + // We have to add 1 to prevent duplication. + let shift = max_id(tt).unwrap_or(0) + 1; + Ok(MacroRules { rules, shift }) } pub fn expand(&self, tt: &tt::Subtree) -> Result { -- cgit v1.2.3 From a5839662f448602d6aa2d724432844fc2d08947e Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 5 Nov 2019 01:16:06 +0800 Subject: Change to add 1 if non zero shift --- crates/ra_mbe/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crates') diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index b92312d52..15f000175 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs @@ -108,7 +108,7 @@ impl MacroRules { // Note that TokenId is started from zero, // We have to add 1 to prevent duplication. - let shift = max_id(tt).unwrap_or(0) + 1; + let shift = max_id(tt).map_or(0, |it| it + 1); Ok(MacroRules { rules, shift }) } -- cgit v1.2.3