diff options
author | bors[bot] <26634292+bors[bot]@users.noreply.github.com> | 2019-11-04 17:21:41 +0000 |
---|---|---|
committer | GitHub <[email protected]> | 2019-11-04 17:21:41 +0000 |
commit | d9fb01f803671bfb6fca6b873bd140d8177ecb1c (patch) | |
tree | 9a1a0b696749d4d079ff0744272f78f497caae29 /crates/ra_mbe | |
parent | cc2d75d0f88bdcb1b3e20db36decb6ee6eca517a (diff) | |
parent | a5839662f448602d6aa2d724432844fc2d08947e (diff) |
Merge #2173
2173: MBE: Add TokenId shift in macro_rules r=matklad a=edwin0cheng
As discussed in #2169 , for fixing duplication TokenId during expansion :
> What we can do here is to re-number the tokens during expansion. Specifically:
> * when we create macro_rules, we note the highest id of the token we have as shift>
> * when we expand macro rules, if we need to output a token from definition, we just re-use its id
> * if we need to output a token from the argument, we increase its id by shift (so it's guaranteed to not to collide with anything from the definition)
> * finally, when we have a HirFileId of the expansion, we can look up the original value of shift and classify node to the arg/def by comparing it's id with shift.
>
This PR implement first 3 points of above solution.
Co-authored-by: Edwin Cheng <[email protected]>
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r-- | crates/ra_mbe/src/lib.rs | 45 | ||||
-rw-r--r-- | crates/ra_mbe/src/tests.rs | 27 |
2 files changed, 70 insertions, 2 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index afdbee84e..15f000175 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs | |||
@@ -42,6 +42,8 @@ pub use crate::syntax_bridge::{ | |||
42 | #[derive(Clone, Debug, PartialEq, Eq)] | 42 | #[derive(Clone, Debug, PartialEq, Eq)] |
43 | pub struct MacroRules { | 43 | pub struct MacroRules { |
44 | pub(crate) rules: Vec<Rule>, | 44 | pub(crate) rules: Vec<Rule>, |
45 | /// Highest id of the token we have in TokenMap | ||
46 | pub(crate) shift: u32, | ||
45 | } | 47 | } |
46 | 48 | ||
47 | #[derive(Clone, Debug, PartialEq, Eq)] | 49 | #[derive(Clone, Debug, PartialEq, Eq)] |
@@ -50,6 +52,38 @@ pub(crate) struct Rule { | |||
50 | pub(crate) rhs: tt::Subtree, | 52 | pub(crate) rhs: tt::Subtree, |
51 | } | 53 | } |
52 | 54 | ||
55 | // Find the max token id inside a subtree | ||
56 | fn max_id(subtree: &tt::Subtree) -> Option<u32> { | ||
57 | subtree | ||
58 | .token_trees | ||
59 | .iter() | ||
60 | .filter_map(|tt| match tt { | ||
61 | tt::TokenTree::Subtree(subtree) => max_id(subtree), | ||
62 | tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) | ||
63 | if ident.id != tt::TokenId::unspecified() => | ||
64 | { | ||
65 | Some(ident.id.0) | ||
66 | } | ||
67 | _ => None, | ||
68 | }) | ||
69 | .max() | ||
70 | } | ||
71 | |||
72 | /// Shift given TokenTree token id | ||
73 | fn shift_subtree(tt: &mut tt::Subtree, shift: u32) { | ||
74 | for t in tt.token_trees.iter_mut() { | ||
75 | match t { | ||
76 | tt::TokenTree::Leaf(leaf) => match leaf { | ||
77 | tt::Leaf::Ident(ident) if ident.id != tt::TokenId::unspecified() => { | ||
78 | ident.id.0 += shift; | ||
79 | } | ||
80 | _ => (), | ||
81 | }, | ||
82 | tt::TokenTree::Subtree(tt) => shift_subtree(tt, shift), | ||
83 | } | ||
84 | } | ||
85 | } | ||
86 | |||
53 | impl MacroRules { | 87 | impl MacroRules { |
54 | pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> { | 88 | pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> { |
55 | // Note: this parsing can be implemented using mbe machinery itself, by | 89 | // Note: this parsing can be implemented using mbe machinery itself, by |
@@ -72,10 +106,17 @@ impl MacroRules { | |||
72 | validate(&rule.lhs)?; | 106 | validate(&rule.lhs)?; |
73 | } | 107 | } |
74 | 108 | ||
75 | Ok(MacroRules { rules }) | 109 | // Note that TokenId is started from zero, |
110 | // We have to add 1 to prevent duplication. | ||
111 | let shift = max_id(tt).map_or(0, |it| it + 1); | ||
112 | Ok(MacroRules { rules, shift }) | ||
76 | } | 113 | } |
114 | |||
77 | pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { | 115 | pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { |
78 | mbe_expander::expand(self, tt) | 116 | // apply shift |
117 | let mut tt = tt.clone(); | ||
118 | shift_subtree(&mut tt, self.shift); | ||
119 | mbe_expander::expand(self, &tt) | ||
79 | } | 120 | } |
80 | } | 121 | } |
81 | 122 | ||
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index f34ab52e1..a23e3afe3 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs | |||
@@ -59,6 +59,33 @@ mod rule_parsing { | |||
59 | // * Make it pass :-) | 59 | // * Make it pass :-) |
60 | 60 | ||
61 | #[test] | 61 | #[test] |
62 | fn test_token_id_shift() { | ||
63 | let macro_definition = r#" | ||
64 | macro_rules! foobar { | ||
65 | ($e:ident) => { foo bar $e } | ||
66 | } | ||
67 | "#; | ||
68 | let rules = create_rules(macro_definition); | ||
69 | let expansion = expand(&rules, "foobar!(baz);"); | ||
70 | |||
71 | fn get_id(t: &tt::TokenTree) -> Option<u32> { | ||
72 | if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = t { | ||
73 | return Some(ident.id.0); | ||
74 | } | ||
75 | None | ||
76 | } | ||
77 | |||
78 | assert_eq!(expansion.token_trees.len(), 3); | ||
79 | // ($e:ident) => { foo bar $e } | ||
80 | // 0 1 2 3 4 | ||
81 | assert_eq!(get_id(&expansion.token_trees[0]), Some(2)); | ||
82 | assert_eq!(get_id(&expansion.token_trees[1]), Some(3)); | ||
83 | |||
84 | // So baz should be 5 | ||
85 | assert_eq!(get_id(&expansion.token_trees[2]), Some(5)); | ||
86 | } | ||
87 | |||
88 | #[test] | ||
62 | fn test_convert_tt() { | 89 | fn test_convert_tt() { |
63 | let macro_definition = r#" | 90 | let macro_definition = r#" |
64 | macro_rules! impl_froms { | 91 | macro_rules! impl_froms { |