diff options
author | bors[bot] <26634292+bors[bot]@users.noreply.github.com> | 2019-09-17 13:53:01 +0100 |
---|---|---|
committer | GitHub <[email protected]> | 2019-09-17 13:53:01 +0100 |
commit | 9421d2a953516b392ae35446bc4f2206dd993c84 (patch) | |
tree | e6c7b46cabe1f10f7da28f3db209df2260045fa8 /crates/ra_mbe/src/mbe_expander/transcriber.rs | |
parent | 8eb2697b7d2a98c952b3acd1711829a13e13cab1 (diff) | |
parent | 4551182f94fe81c314f79ddf8916a5520cfd03b0 (diff) |
Merge #1858
1858: use usual token tree for macro expansions r=matklad a=matklad
Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_mbe/src/mbe_expander/transcriber.rs')
-rw-r--r-- | crates/ra_mbe/src/mbe_expander/transcriber.rs | 288 |
1 files changed, 146 insertions, 142 deletions
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index a3df1b7de..c22680b93 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs | |||
@@ -1,16 +1,20 @@ | |||
1 | //! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like | ||
2 | //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` | ||
3 | |||
1 | use ra_syntax::SmolStr; | 4 | use ra_syntax::SmolStr; |
2 | 5 | ||
3 | use crate::{ | 6 | use crate::{ |
4 | mbe_expander::{Binding, Bindings, Fragment}, | 7 | mbe_expander::{Binding, Bindings, Fragment}, |
8 | parser::{parse_template, Op, RepeatKind, Separator}, | ||
5 | ExpandError, | 9 | ExpandError, |
6 | }; | 10 | }; |
7 | 11 | ||
8 | impl Bindings { | 12 | impl Bindings { |
9 | fn contains(&self, name: &SmolStr) -> bool { | 13 | fn contains(&self, name: &str) -> bool { |
10 | self.inner.contains_key(name) | 14 | self.inner.contains_key(name) |
11 | } | 15 | } |
12 | 16 | ||
13 | fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { | 17 | fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> { |
14 | let mut b = self.inner.get(name).ok_or_else(|| { | 18 | let mut b = self.inner.get(name).ok_or_else(|| { |
15 | ExpandError::BindingError(format!("could not find binding `{}`", name)) | 19 | ExpandError::BindingError(format!("could not find binding `{}`", name)) |
16 | })?; | 20 | })?; |
@@ -43,11 +47,12 @@ impl Bindings { | |||
43 | } | 47 | } |
44 | 48 | ||
45 | pub(super) fn transcribe( | 49 | pub(super) fn transcribe( |
50 | template: &tt::Subtree, | ||
46 | bindings: &Bindings, | 51 | bindings: &Bindings, |
47 | template: &crate::Subtree, | ||
48 | ) -> Result<tt::Subtree, ExpandError> { | 52 | ) -> Result<tt::Subtree, ExpandError> { |
53 | assert!(template.delimiter == tt::Delimiter::None); | ||
49 | let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; | 54 | let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; |
50 | expand_subtree(template, &mut ctx) | 55 | expand_subtree(&mut ctx, template) |
51 | } | 56 | } |
52 | 57 | ||
53 | #[derive(Debug)] | 58 | #[derive(Debug)] |
@@ -57,159 +62,158 @@ struct ExpandCtx<'a> { | |||
57 | var_expanded: bool, | 62 | var_expanded: bool, |
58 | } | 63 | } |
59 | 64 | ||
60 | fn expand_subtree( | 65 | fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { |
61 | template: &crate::Subtree, | ||
62 | ctx: &mut ExpandCtx, | ||
63 | ) -> Result<tt::Subtree, ExpandError> { | ||
64 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | 66 | let mut buf: Vec<tt::TokenTree> = Vec::new(); |
65 | for tt in template.token_trees.iter() { | 67 | for op in parse_template(template) { |
66 | let tt = expand_tt(tt, ctx)?; | 68 | match op? { |
67 | push_fragment(&mut buf, tt); | 69 | Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), |
70 | Op::TokenTree(tt::TokenTree::Subtree(tt)) => { | ||
71 | let tt = expand_subtree(ctx, tt)?; | ||
72 | buf.push(tt.into()); | ||
73 | } | ||
74 | Op::Var { name, kind: _ } => { | ||
75 | let fragment = expand_var(ctx, name)?; | ||
76 | push_fragment(&mut buf, fragment); | ||
77 | } | ||
78 | Op::Repeat { subtree, kind, separator } => { | ||
79 | let fragment = expand_repeat(ctx, subtree, kind, separator)?; | ||
80 | push_fragment(&mut buf, fragment) | ||
81 | } | ||
82 | } | ||
68 | } | 83 | } |
69 | |||
70 | Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) | 84 | Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) |
71 | } | 85 | } |
72 | 86 | ||
73 | fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> { | 87 | fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> { |
74 | let res: tt::TokenTree = match template { | 88 | let res = if v == "crate" { |
75 | crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), | 89 | // FIXME: Properly handle $crate token |
76 | crate::TokenTree::Repeat(repeat) => { | 90 | let tt = |
77 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | 91 | tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) |
78 | ctx.nesting.push(0); | 92 | .into(); |
79 | // Dirty hack to make macro-expansion terminate. | 93 | Fragment::Tokens(tt) |
80 | // This should be replaced by a propper macro-by-example implementation | 94 | } else if !ctx.bindings.contains(v) { |
81 | let mut limit = 65536; | 95 | // Note that it is possible to have a `$var` inside a macro which is not bound. |
82 | let mut has_seps = 0; | 96 | // For example: |
83 | let mut counter = 0; | 97 | // ``` |
84 | 98 | // macro_rules! foo { | |
85 | // We store the old var expanded value, and restore it later | 99 | // ($a:ident, $b:ident, $c:tt) => { |
86 | // It is because before this `$repeat`, | 100 | // macro_rules! bar { |
87 | // it is possible some variables already expanad in the same subtree | 101 | // ($bi:ident) => { |
88 | // | 102 | // fn $bi() -> u8 {$c} |
89 | // `some_var_expanded` keep check if the deeper subtree has expanded variables | 103 | // } |
90 | let mut some_var_expanded = false; | 104 | // } |
91 | let old_var_expanded = ctx.var_expanded; | 105 | // } |
92 | ctx.var_expanded = false; | 106 | // ``` |
93 | 107 | // We just treat it a normal tokens | |
94 | while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { | 108 | let tt = tt::Subtree { |
95 | // if no var expanded in the child, we count it as a fail | 109 | delimiter: tt::Delimiter::None, |
96 | if !ctx.var_expanded { | 110 | token_trees: vec![ |
97 | break; | 111 | tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), |
98 | } | 112 | tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) |
113 | .into(), | ||
114 | ], | ||
115 | } | ||
116 | .into(); | ||
117 | Fragment::Tokens(tt) | ||
118 | } else { | ||
119 | let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone(); | ||
120 | ctx.var_expanded = true; | ||
121 | fragment | ||
122 | }; | ||
123 | Ok(res) | ||
124 | } | ||
99 | 125 | ||
100 | // Reset `ctx.var_expandeded` to see if there is other expanded variable | 126 | fn expand_repeat( |
101 | // in the next matching | 127 | ctx: &mut ExpandCtx, |
102 | some_var_expanded = true; | 128 | template: &tt::Subtree, |
103 | ctx.var_expanded = false; | 129 | kind: RepeatKind, |
104 | 130 | separator: Option<Separator>, | |
105 | counter += 1; | 131 | ) -> Result<Fragment, ExpandError> { |
106 | limit -= 1; | 132 | let mut buf: Vec<tt::TokenTree> = Vec::new(); |
107 | if limit == 0 { | 133 | ctx.nesting.push(0); |
108 | log::warn!( | 134 | // Dirty hack to make macro-expansion terminate. |
109 | "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", | 135 | // This should be replaced by a propper macro-by-example implementation |
110 | template, | 136 | let mut limit = 65536; |
111 | ctx | 137 | let mut has_seps = 0; |
112 | ); | 138 | let mut counter = 0; |
113 | break; | 139 | |
114 | } | 140 | // We store the old var expanded value, and restore it later |
141 | // It is because before this `$repeat`, | ||
142 | // it is possible some variables already expanad in the same subtree | ||
143 | // | ||
144 | // `some_var_expanded` keep check if the deeper subtree has expanded variables | ||
145 | let mut some_var_expanded = false; | ||
146 | let old_var_expanded = ctx.var_expanded; | ||
147 | ctx.var_expanded = false; | ||
148 | |||
149 | while let Ok(mut t) = expand_subtree(ctx, template) { | ||
150 | t.delimiter = tt::Delimiter::None; | ||
151 | // if no var expanded in the child, we count it as a fail | ||
152 | if !ctx.var_expanded { | ||
153 | break; | ||
154 | } | ||
115 | 155 | ||
116 | let idx = ctx.nesting.pop().unwrap(); | 156 | // Reset `ctx.var_expandeded` to see if there is other expanded variable |
117 | ctx.nesting.push(idx + 1); | 157 | // in the next matching |
118 | push_subtree(&mut buf, t); | 158 | some_var_expanded = true; |
119 | 159 | ctx.var_expanded = false; | |
120 | if let Some(ref sep) = repeat.separator { | 160 | |
121 | match sep { | 161 | counter += 1; |
122 | crate::Separator::Ident(ident) => { | 162 | limit -= 1; |
123 | has_seps = 1; | 163 | if limit == 0 { |
124 | buf.push(tt::Leaf::from(ident.clone()).into()); | 164 | log::warn!( |
125 | } | 165 | "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", |
126 | crate::Separator::Literal(lit) => { | 166 | template, |
127 | has_seps = 1; | 167 | ctx |
128 | buf.push(tt::Leaf::from(lit.clone()).into()); | 168 | ); |
129 | } | 169 | break; |
130 | 170 | } | |
131 | crate::Separator::Puncts(puncts) => { | 171 | |
132 | has_seps = puncts.len(); | 172 | let idx = ctx.nesting.pop().unwrap(); |
133 | for punct in puncts { | 173 | ctx.nesting.push(idx + 1); |
134 | buf.push(tt::Leaf::from(*punct).into()); | 174 | push_subtree(&mut buf, t); |
135 | } | 175 | |
136 | } | 176 | if let Some(ref sep) = separator { |
137 | } | 177 | match sep { |
178 | Separator::Ident(ident) => { | ||
179 | has_seps = 1; | ||
180 | buf.push(tt::Leaf::from(ident.clone()).into()); | ||
181 | } | ||
182 | Separator::Literal(lit) => { | ||
183 | has_seps = 1; | ||
184 | buf.push(tt::Leaf::from(lit.clone()).into()); | ||
138 | } | 185 | } |
139 | 186 | ||
140 | if let crate::RepeatKind::ZeroOrOne = repeat.kind { | 187 | Separator::Puncts(puncts) => { |
141 | break; | 188 | has_seps = puncts.len(); |
189 | for punct in puncts { | ||
190 | buf.push(tt::Leaf::from(*punct).into()); | ||
191 | } | ||
142 | } | 192 | } |
143 | } | 193 | } |
194 | } | ||
144 | 195 | ||
145 | // Restore the `var_expanded` by combining old one and the new one | 196 | if RepeatKind::ZeroOrOne == kind { |
146 | ctx.var_expanded = some_var_expanded || old_var_expanded; | 197 | break; |
198 | } | ||
199 | } | ||
147 | 200 | ||
148 | ctx.nesting.pop().unwrap(); | 201 | // Restore the `var_expanded` by combining old one and the new one |
149 | for _ in 0..has_seps { | 202 | ctx.var_expanded = some_var_expanded || old_var_expanded; |
150 | buf.pop(); | ||
151 | } | ||
152 | 203 | ||
153 | if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { | 204 | ctx.nesting.pop().unwrap(); |
154 | return Err(ExpandError::UnexpectedToken); | 205 | for _ in 0..has_seps { |
155 | } | 206 | buf.pop(); |
207 | } | ||
156 | 208 | ||
157 | // Check if it is a single token subtree without any delimiter | 209 | if RepeatKind::OneOrMore == kind && counter == 0 { |
158 | // e.g {Delimiter:None> ['>'] /Delimiter:None>} | 210 | return Err(ExpandError::UnexpectedToken); |
159 | tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() | 211 | } |
160 | } | 212 | |
161 | crate::TokenTree::Leaf(leaf) => match leaf { | 213 | // Check if it is a single token subtree without any delimiter |
162 | crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident { | 214 | // e.g {Delimiter:None> ['>'] /Delimiter:None>} |
163 | text: ident.text.clone(), | 215 | let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into(); |
164 | id: tt::TokenId::unspecified(), | 216 | Ok(Fragment::Tokens(tt)) |
165 | }) | ||
166 | .into(), | ||
167 | crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(), | ||
168 | crate::Leaf::Var(v) => { | ||
169 | if v.text == "crate" { | ||
170 | // FIXME: Properly handle $crate token | ||
171 | tt::Leaf::from(tt::Ident { | ||
172 | text: "$crate".into(), | ||
173 | id: tt::TokenId::unspecified(), | ||
174 | }) | ||
175 | .into() | ||
176 | } else if !ctx.bindings.contains(&v.text) { | ||
177 | // Note that it is possible to have a `$var` inside a macro which is not bound. | ||
178 | // For example: | ||
179 | // ``` | ||
180 | // macro_rules! foo { | ||
181 | // ($a:ident, $b:ident, $c:tt) => { | ||
182 | // macro_rules! bar { | ||
183 | // ($bi:ident) => { | ||
184 | // fn $bi() -> u8 {$c} | ||
185 | // } | ||
186 | // } | ||
187 | // } | ||
188 | // ``` | ||
189 | // We just treat it a normal tokens | ||
190 | tt::Subtree { | ||
191 | delimiter: tt::Delimiter::None, | ||
192 | token_trees: vec![ | ||
193 | tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }) | ||
194 | .into(), | ||
195 | tt::Leaf::from(tt::Ident { | ||
196 | text: v.text.clone(), | ||
197 | id: tt::TokenId::unspecified(), | ||
198 | }) | ||
199 | .into(), | ||
200 | ], | ||
201 | } | ||
202 | .into() | ||
203 | } else { | ||
204 | let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); | ||
205 | ctx.var_expanded = true; | ||
206 | return Ok(fragment); | ||
207 | } | ||
208 | } | ||
209 | crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(), | ||
210 | }, | ||
211 | }; | ||
212 | Ok(Fragment::Tokens(res)) | ||
213 | } | 217 | } |
214 | 218 | ||
215 | fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { | 219 | fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { |