aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2019-09-17 13:53:01 +0100
committerGitHub <[email protected]>2019-09-17 13:53:01 +0100
commit9421d2a953516b392ae35446bc4f2206dd993c84 (patch)
treee6c7b46cabe1f10f7da28f3db209df2260045fa8
parent8eb2697b7d2a98c952b3acd1711829a13e13cab1 (diff)
parent4551182f94fe81c314f79ddf8916a5520cfd03b0 (diff)
Merge #1858
1858: use usual token tree for macro expansions r=matklad a=matklad Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat
-rw-r--r--crates/ra_mbe/src/lib.rs157
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs19
-rw-r--r--crates/ra_mbe/src/mbe_expander/matcher.rs361
-rw-r--r--crates/ra_mbe/src/mbe_expander/transcriber.rs288
-rw-r--r--crates/ra_mbe/src/mbe_parser.rs187
-rw-r--r--crates/ra_mbe/src/parser.rs187
-rw-r--r--crates/ra_mbe/src/subtree_parser.rs91
-rw-r--r--crates/ra_mbe/src/tests.rs48
-rw-r--r--crates/ra_mbe/src/tt_cursor.rs281
-rw-r--r--crates/ra_mbe/src/tt_iter.rs67
10 files changed, 753 insertions, 933 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index f07f000ff..41720df79 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -3,30 +3,19 @@
3/// interface, although it contains some code to bridge `SyntaxNode`s and 3/// interface, although it contains some code to bridge `SyntaxNode`s and
4/// `TokenTree`s as well! 4/// `TokenTree`s as well!
5 5
6macro_rules! impl_froms { 6mod parser;
7 ($e:ident: $($v:ident), *) => {
8 $(
9 impl From<$v> for $e {
10 fn from(it: $v) -> $e {
11 $e::$v(it)
12 }
13 }
14 )*
15 }
16}
17
18mod mbe_parser;
19mod mbe_expander; 7mod mbe_expander;
20mod syntax_bridge; 8mod syntax_bridge;
21mod tt_cursor; 9mod tt_iter;
22mod subtree_source; 10mod subtree_source;
23mod subtree_parser;
24
25use ra_syntax::SmolStr;
26use smallvec::SmallVec;
27 11
28pub use tt::{Delimiter, Punct}; 12pub use tt::{Delimiter, Punct};
29 13
14use crate::{
15 parser::{parse_pattern, Op},
16 tt_iter::TtIter,
17};
18
30#[derive(Debug, PartialEq, Eq)] 19#[derive(Debug, PartialEq, Eq)]
31pub enum ParseError { 20pub enum ParseError {
32 Expected(String), 21 Expected(String),
@@ -38,6 +27,7 @@ pub enum ExpandError {
38 UnexpectedToken, 27 UnexpectedToken,
39 BindingError(String), 28 BindingError(String),
40 ConversionError, 29 ConversionError,
30 InvalidRepeat,
41} 31}
42 32
43pub use crate::syntax_bridge::{ 33pub use crate::syntax_bridge::{
@@ -54,97 +44,72 @@ pub struct MacroRules {
54 pub(crate) rules: Vec<Rule>, 44 pub(crate) rules: Vec<Rule>,
55} 45}
56 46
47#[derive(Clone, Debug, PartialEq, Eq)]
48pub(crate) struct Rule {
49 pub(crate) lhs: tt::Subtree,
50 pub(crate) rhs: tt::Subtree,
51}
52
57impl MacroRules { 53impl MacroRules {
58 pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> { 54 pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
59 mbe_parser::parse(tt) 55 let mut src = TtIter::new(tt);
56 let mut rules = Vec::new();
57 while src.len() > 0 {
58 let rule = Rule::parse(&mut src)?;
59 rules.push(rule);
60 if let Err(()) = src.expect_char(';') {
61 if src.len() > 0 {
62 return Err(ParseError::Expected("expected `:`".to_string()));
63 }
64 break;
65 }
66 }
67 Ok(MacroRules { rules })
60 } 68 }
61 pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 69 pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
62 mbe_expander::expand(self, tt) 70 mbe_expander::expand(self, tt)
63 } 71 }
64} 72}
65 73
66#[derive(Clone, Debug, PartialEq, Eq)] 74impl Rule {
67pub(crate) struct Rule { 75 fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
68 pub(crate) lhs: Subtree, 76 let mut lhs = src
69 pub(crate) rhs: Subtree, 77 .expect_subtree()
70} 78 .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
71 79 .clone();
72#[derive(Clone, Debug, PartialEq, Eq)] 80 validate(&lhs)?;
73pub(crate) enum TokenTree { 81 lhs.delimiter = tt::Delimiter::None;
74 Leaf(Leaf), 82 src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
75 Subtree(Subtree), 83 src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
76 Repeat(Repeat), 84 let mut rhs = src
77} 85 .expect_subtree()
78impl_froms!(TokenTree: Leaf, Subtree, Repeat); 86 .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
79 87 .clone();
80#[derive(Clone, Debug, PartialEq, Eq)] 88 rhs.delimiter = tt::Delimiter::None;
81pub(crate) enum Leaf { 89 Ok(crate::Rule { lhs, rhs })
82 Literal(Literal), 90 }
83 Punct(Punct),
84 Ident(Ident),
85 Var(Var),
86}
87impl_froms!(Leaf: Literal, Punct, Ident, Var);
88
89#[derive(Clone, Debug, PartialEq, Eq)]
90pub(crate) struct Subtree {
91 pub(crate) delimiter: Delimiter,
92 pub(crate) token_trees: Vec<TokenTree>,
93}
94
95#[derive(Clone, Debug, Eq)]
96pub(crate) enum Separator {
97 Literal(tt::Literal),
98 Ident(tt::Ident),
99 Puncts(SmallVec<[tt::Punct; 3]>),
100} 91}
101 92
102// Note that when we compare a Separator, we just care about its textual value. 93fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
103impl PartialEq for crate::Separator { 94 for op in parse_pattern(pattern) {
104 fn eq(&self, other: &crate::Separator) -> bool { 95 let op = match op {
105 use crate::Separator::*; 96 Ok(it) => it,
106 97 Err(e) => {
107 match (self, other) { 98 let msg = match e {
108 (Ident(ref a), Ident(ref b)) => a.text == b.text, 99 ExpandError::InvalidRepeat => "invalid repeat".to_string(),
109 (Literal(ref a), Literal(ref b)) => a.text == b.text, 100 _ => "invalid macro definition".to_string(),
110 (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { 101 };
111 let a_iter = a.iter().map(|a| a.char); 102 return Err(ParseError::Expected(msg));
112 let b_iter = b.iter().map(|b| b.char); 103 }
113 a_iter.eq(b_iter) 104 };
105 match op {
106 Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => {
107 validate(subtree)?
114 } 108 }
115 _ => false, 109 _ => (),
116 } 110 }
117 } 111 }
118} 112 Ok(())
119
120#[derive(Clone, Debug, PartialEq, Eq)]
121pub(crate) struct Repeat {
122 pub(crate) subtree: Subtree,
123 pub(crate) kind: RepeatKind,
124 pub(crate) separator: Option<Separator>,
125}
126
127#[derive(Clone, Debug, PartialEq, Eq)]
128pub(crate) enum RepeatKind {
129 ZeroOrMore,
130 OneOrMore,
131 ZeroOrOne,
132}
133
134#[derive(Clone, Debug, PartialEq, Eq)]
135pub(crate) struct Literal {
136 pub(crate) text: SmolStr,
137}
138
139#[derive(Clone, Debug, PartialEq, Eq)]
140pub(crate) struct Ident {
141 pub(crate) text: SmolStr,
142}
143
144#[derive(Clone, Debug, PartialEq, Eq)]
145pub(crate) struct Var {
146 pub(crate) text: SmolStr,
147 pub(crate) kind: Option<SmolStr>,
148} 113}
149 114
150#[cfg(test)] 115#[cfg(test)]
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 15d9d83e2..b455b7321 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -8,7 +8,6 @@ mod transcriber;
8use ra_syntax::SmolStr; 8use ra_syntax::SmolStr;
9use rustc_hash::FxHashMap; 9use rustc_hash::FxHashMap;
10 10
11use crate::tt_cursor::TtCursor;
12use crate::ExpandError; 11use crate::ExpandError;
13 12
14pub(crate) fn expand( 13pub(crate) fn expand(
@@ -19,12 +18,8 @@ pub(crate) fn expand(
19} 18}
20 19
21fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 20fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
22 let mut input = TtCursor::new(input); 21 let bindings = matcher::match_(&rule.lhs, input)?;
23 let bindings = matcher::match_lhs(&rule.lhs, &mut input)?; 22 let res = transcriber::transcribe(&rule.rhs, &bindings)?;
24 if !input.is_eof() {
25 return Err(ExpandError::UnexpectedToken);
26 }
27 let res = transcriber::transcribe(&bindings, &rule.rhs)?;
28 Ok(res) 23 Ok(res)
29} 24}
30 25
@@ -103,13 +98,6 @@ mod tests {
103 98
104 #[test] 99 #[test]
105 fn test_expand_rule() { 100 fn test_expand_rule() {
106 // FIXME: The missing $var check should be in parsing phase
107 // assert_err(
108 // "($i:ident) => ($j)",
109 // "foo!{a}",
110 // ExpandError::BindingError(String::from("could not find binding `j`")),
111 // );
112
113 assert_err( 101 assert_err(
114 "($($i:ident);*) => ($i)", 102 "($($i:ident);*) => ($i)",
115 "foo!{a}", 103 "foo!{a}",
@@ -118,9 +106,6 @@ mod tests {
118 )), 106 )),
119 ); 107 );
120 108
121 assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
122 assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken);
123
124 // FIXME: 109 // FIXME:
125 // Add an err test case for ($($i:ident)) => ($()) 110 // Add an err test case for ($($i:ident)) => ($())
126 } 111 }
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 100a3b0e0..aff953102 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -1,11 +1,14 @@
1use crate::{ 1use crate::{
2 mbe_expander::{Binding, Bindings, Fragment}, 2 mbe_expander::{Binding, Bindings, Fragment},
3 tt_cursor::TtCursor, 3 parser::{parse_pattern, Op, RepeatKind, Separator},
4 subtree_source::SubtreeTokenSource,
5 tt_iter::TtIter,
4 ExpandError, 6 ExpandError,
5}; 7};
6 8
7use ra_parser::FragmentKind::*; 9use ra_parser::{FragmentKind::*, TreeSink};
8use ra_syntax::SmolStr; 10use ra_syntax::{SmolStr, SyntaxKind};
11use tt::buffer::{Cursor, TokenBuffer};
9 12
10impl Bindings { 13impl Bindings {
11 fn push_optional(&mut self, name: &SmolStr) { 14 fn push_optional(&mut self, name: &SmolStr) {
@@ -42,121 +45,247 @@ impl Bindings {
42 } 45 }
43 Ok(()) 46 Ok(())
44 } 47 }
48}
45 49
46 fn merge(&mut self, nested: Bindings) { 50macro_rules! err {
47 self.inner.extend(nested.inner); 51 () => {
48 } 52 ExpandError::BindingError(format!(""))
53 };
54 ($($tt:tt)*) => {
55 ExpandError::BindingError(format!($($tt)*))
56 };
49} 57}
50 58
51pub(super) fn match_lhs( 59macro_rules! bail {
52 pattern: &crate::Subtree, 60 ($($tt:tt)*) => {
53 input: &mut TtCursor, 61 return Err(err!($($tt)*))
54) -> Result<Bindings, ExpandError> { 62 };
63}
64
65pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Bindings, ExpandError> {
66 assert!(pattern.delimiter == tt::Delimiter::None);
67
55 let mut res = Bindings::default(); 68 let mut res = Bindings::default();
56 for pat in pattern.token_trees.iter() { 69 let mut src = TtIter::new(src);
57 match pat { 70
58 crate::TokenTree::Leaf(leaf) => match leaf { 71 match_subtree(&mut res, pattern, &mut src)?;
59 crate::Leaf::Var(crate::Var { text, kind }) => { 72
60 let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; 73 if src.len() > 0 {
61 match match_meta_var(kind.as_str(), input)? { 74 bail!("leftover tokens");
62 Some(fragment) => { 75 }
63 res.inner.insert(text.clone(), Binding::Fragment(fragment)); 76
64 } 77 Ok(res)
65 None => res.push_optional(text), 78}
66 } 79
80fn match_subtree(
81 bindings: &mut Bindings,
82 pattern: &tt::Subtree,
83 src: &mut TtIter,
84) -> Result<(), ExpandError> {
85 for op in parse_pattern(pattern) {
86 match op? {
87 Op::TokenTree(tt::TokenTree::Leaf(lhs)) => {
88 let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?;
89 match (lhs, rhs) {
90 (
91 tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
92 tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
93 ) if lhs == rhs => (),
94 (
95 tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
96 tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
97 ) if lhs == rhs => (),
98 (
99 tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
100 tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
101 ) if lhs == rhs => (),
102 _ => Err(ExpandError::UnexpectedToken)?,
67 } 103 }
68 crate::Leaf::Punct(punct) => { 104 }
69 if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) { 105 Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
70 return Err(ExpandError::UnexpectedToken); 106 let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?;
71 } 107 if lhs.delimiter != rhs.delimiter {
108 bail!("mismatched delimiter")
72 } 109 }
73 crate::Leaf::Ident(ident) => { 110 let mut src = TtIter::new(rhs);
74 if input.eat_ident().map(|i| &i.text) != Some(&ident.text) { 111 match_subtree(bindings, lhs, &mut src)?;
75 return Err(ExpandError::UnexpectedToken); 112 if src.len() > 0 {
76 } 113 bail!("leftover tokens");
77 } 114 }
78 crate::Leaf::Literal(literal) => { 115 }
79 if input.eat_literal().map(|i| &i.text) != Some(&literal.text) { 116 Op::Var { name, kind } => {
80 return Err(ExpandError::UnexpectedToken); 117 let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?;
118 match match_meta_var(kind.as_str(), src)? {
119 Some(fragment) => {
120 bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
81 } 121 }
122 None => bindings.push_optional(name),
82 } 123 }
124 ()
125 }
126 Op::Repeat { subtree, kind, separator } => {
127 match_repeat(bindings, subtree, kind, separator, src)?
128 }
129 }
130 }
131 Ok(())
132}
133
134impl<'a> TtIter<'a> {
135 fn eat_separator(&mut self, separator: &Separator) -> bool {
136 let mut fork = self.clone();
137 let ok = match separator {
138 Separator::Ident(lhs) => match fork.expect_ident() {
139 Ok(rhs) => rhs.text == lhs.text,
140 _ => false,
83 }, 141 },
84 crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => { 142 Separator::Literal(lhs) => match fork.expect_literal() {
85 // Dirty hack to make macro-expansion terminate. 143 Ok(rhs) => rhs.text == lhs.text,
86 // This should be replaced by a propper macro-by-example implementation 144 _ => false,
87 let mut limit = 65536; 145 },
88 let mut counter = 0; 146 Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
89 147 Ok(rhs) => rhs.char == lhs.char,
90 let mut memento = input.save(); 148 _ => false,
91 149 }),
92 loop { 150 };
93 match match_lhs(subtree, input) { 151 if ok {
94 Ok(nested) => { 152 *self = fork;
95 limit -= 1; 153 }
96 if limit == 0 { 154 ok
97 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); 155 }
98 break;
99 }
100
101 memento = input.save();
102 res.push_nested(counter, nested)?;
103 counter += 1;
104 if counter == 1 {
105 if let crate::RepeatKind::ZeroOrOne = kind {
106 break;
107 }
108 }
109
110 if let Some(separator) = separator {
111 if !input
112 .eat_seperator()
113 .map(|sep| sep == *separator)
114 .unwrap_or(false)
115 {
116 input.rollback(memento);
117 break;
118 }
119 }
120 }
121 Err(_) => {
122 input.rollback(memento);
123 break;
124 }
125 }
126 }
127 156
128 match kind { 157 pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
129 crate::RepeatKind::OneOrMore if counter == 0 => { 158 let ident = self.expect_ident()?;
130 return Err(ExpandError::UnexpectedToken); 159 // check if it start from "`"
131 } 160 if ident.text.chars().next() != Some('\'') {
132 _ if counter == 0 => { 161 return Err(());
133 // Collect all empty variables in subtrees 162 }
134 collect_vars(subtree).iter().for_each(|s| res.push_empty(s)); 163 Ok(ident)
135 } 164 }
136 _ => {} 165
166 pub(crate) fn expect_fragment(
167 &mut self,
168 fragment_kind: ra_parser::FragmentKind,
169 ) -> Result<tt::TokenTree, ()> {
170 pub(crate) struct OffsetTokenSink<'a> {
171 pub(crate) cursor: Cursor<'a>,
172 pub(crate) error: bool,
173 }
174
175 impl<'a> TreeSink for OffsetTokenSink<'a> {
176 fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
177 for _ in 0..n_tokens {
178 self.cursor = self.cursor.bump_subtree();
137 } 179 }
138 } 180 }
139 crate::TokenTree::Subtree(subtree) => { 181 fn start_node(&mut self, _kind: SyntaxKind) {}
140 let input_subtree = 182 fn finish_node(&mut self) {}
141 input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?; 183 fn error(&mut self, _error: ra_parser::ParseError) {
142 if subtree.delimiter != input_subtree.delimiter { 184 self.error = true;
143 return Err(ExpandError::UnexpectedToken); 185 }
186 }
187
188 let buffer = TokenBuffer::new(self.inner.as_slice());
189 let mut src = SubtreeTokenSource::new(&buffer);
190 let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
191
192 ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind);
193
194 if !sink.cursor.is_root() || sink.error {
195 return Err(());
196 }
197
198 let mut curr = buffer.begin();
199 let mut res = vec![];
200
201 while curr != sink.cursor {
202 if let Some(token) = curr.token_tree() {
203 res.push(token);
204 }
205 curr = curr.bump();
206 }
207 self.inner = self.inner.as_slice()[res.len()..].iter();
208 match res.len() {
209 0 => Err(()),
210 1 => Ok(res[0].clone()),
211 _ => Ok(tt::TokenTree::Subtree(tt::Subtree {
212 delimiter: tt::Delimiter::None,
213 token_trees: res.into_iter().cloned().collect(),
214 })),
215 }
216 }
217
218 pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
219 let mut fork = self.clone();
220 match fork.expect_fragment(Visibility) {
221 Ok(tt) => {
222 *self = fork;
223 Some(tt)
224 }
225 Err(()) => None,
226 }
227 }
228}
229
230pub(super) fn match_repeat(
231 bindings: &mut Bindings,
232 pattern: &tt::Subtree,
233 kind: RepeatKind,
234 separator: Option<Separator>,
235 src: &mut TtIter,
236) -> Result<(), ExpandError> {
237 // Dirty hack to make macro-expansion terminate.
238 // This should be replaced by a propper macro-by-example implementation
239 let mut limit = 65536;
240 let mut counter = 0;
241
242 for i in 0.. {
243 let mut fork = src.clone();
244
245 if let Some(separator) = &separator {
246 if i != 0 && !fork.eat_separator(separator) {
247 break;
248 }
249 }
250
251 let mut nested = Bindings::default();
252 match match_subtree(&mut nested, pattern, &mut fork) {
253 Ok(()) => {
254 limit -= 1;
255 if limit == 0 {
256 log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator);
257 break;
144 } 258 }
259 *src = fork;
145 260
146 let mut input = TtCursor::new(input_subtree); 261 bindings.push_nested(counter, nested)?;
147 let bindings = match_lhs(&subtree, &mut input)?; 262 counter += 1;
148 if !input.is_eof() { 263 if counter == 1 {
149 return Err(ExpandError::UnexpectedToken); 264 if let RepeatKind::ZeroOrOne = kind {
265 break;
266 }
150 } 267 }
268 }
269 Err(_) => break,
270 }
271 }
151 272
152 res.merge(bindings); 273 match (kind, counter) {
274 (RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken),
275 (_, 0) => {
276 // Collect all empty variables in subtrees
277 let mut vars = Vec::new();
278 collect_vars(&mut vars, pattern)?;
279 for var in vars {
280 bindings.push_empty(&var)
153 } 281 }
154 } 282 }
283 _ => (),
155 } 284 }
156 Ok(res) 285 Ok(())
157} 286}
158 287
159fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> { 288fn match_meta_var(kind: &str, input: &mut TtIter) -> Result<Option<Fragment>, ExpandError> {
160 let fragment = match kind { 289 let fragment = match kind {
161 "path" => Path, 290 "path" => Path,
162 "expr" => Expr, 291 "expr" => Expr,
@@ -169,17 +298,20 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
169 _ => { 298 _ => {
170 let tt = match kind { 299 let tt = match kind {
171 "ident" => { 300 "ident" => {
172 let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone(); 301 let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone();
173 tt::Leaf::from(ident).into() 302 tt::Leaf::from(ident).into()
174 } 303 }
175 "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(), 304 "tt" => input.next().ok_or_else(|| err!())?.clone(),
176 "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(), 305 "lifetime" => {
306 let ident = input.expect_lifetime().map_err(|()| err!())?;
307 tt::Leaf::Ident(ident.clone()).into()
308 }
177 "literal" => { 309 "literal" => {
178 let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); 310 let literal = input.expect_literal().map_err(|()| err!())?.clone();
179 tt::Leaf::from(literal).into() 311 tt::Leaf::from(literal).into()
180 } 312 }
181 // `vis` is optional 313 // `vis` is optional
182 "vis" => match input.try_eat_vis() { 314 "vis" => match input.eat_vis() {
183 Some(vis) => vis, 315 Some(vis) => vis,
184 None => return Ok(None), 316 None => return Ok(None),
185 }, 317 },
@@ -188,28 +320,19 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>,
188 return Ok(Some(Fragment::Tokens(tt))); 320 return Ok(Some(Fragment::Tokens(tt)));
189 } 321 }
190 }; 322 };
191 let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?; 323 let tt = input.expect_fragment(fragment).map_err(|()| err!())?;
192 let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; 324 let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) };
193 Ok(Some(fragment)) 325 Ok(Some(fragment))
194} 326}
195 327
196fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> { 328fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> {
197 let mut res = Vec::new(); 329 for op in parse_pattern(pattern) {
198 330 match op? {
199 for tkn in subtree.token_trees.iter() { 331 Op::Var { name, .. } => buf.push(name.clone()),
200 match tkn { 332 Op::TokenTree(tt::TokenTree::Leaf(_)) => (),
201 crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => { 333 Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?,
202 res.push(text.clone()); 334 Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
203 }
204 crate::TokenTree::Subtree(subtree) => {
205 res.extend(collect_vars(subtree));
206 }
207 crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => {
208 res.extend(collect_vars(subtree));
209 }
210 _ => {}
211 } 335 }
212 } 336 }
213 337 Ok(())
214 res
215} 338}
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs
index a3df1b7de..c22680b93 100644
--- a/crates/ra_mbe/src/mbe_expander/transcriber.rs
+++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs
@@ -1,16 +1,20 @@
1//! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like
2//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
3
1use ra_syntax::SmolStr; 4use ra_syntax::SmolStr;
2 5
3use crate::{ 6use crate::{
4 mbe_expander::{Binding, Bindings, Fragment}, 7 mbe_expander::{Binding, Bindings, Fragment},
8 parser::{parse_template, Op, RepeatKind, Separator},
5 ExpandError, 9 ExpandError,
6}; 10};
7 11
8impl Bindings { 12impl Bindings {
9 fn contains(&self, name: &SmolStr) -> bool { 13 fn contains(&self, name: &str) -> bool {
10 self.inner.contains_key(name) 14 self.inner.contains_key(name)
11 } 15 }
12 16
13 fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { 17 fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> {
14 let mut b = self.inner.get(name).ok_or_else(|| { 18 let mut b = self.inner.get(name).ok_or_else(|| {
15 ExpandError::BindingError(format!("could not find binding `{}`", name)) 19 ExpandError::BindingError(format!("could not find binding `{}`", name))
16 })?; 20 })?;
@@ -43,11 +47,12 @@ impl Bindings {
43} 47}
44 48
45pub(super) fn transcribe( 49pub(super) fn transcribe(
50 template: &tt::Subtree,
46 bindings: &Bindings, 51 bindings: &Bindings,
47 template: &crate::Subtree,
48) -> Result<tt::Subtree, ExpandError> { 52) -> Result<tt::Subtree, ExpandError> {
53 assert!(template.delimiter == tt::Delimiter::None);
49 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; 54 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false };
50 expand_subtree(template, &mut ctx) 55 expand_subtree(&mut ctx, template)
51} 56}
52 57
53#[derive(Debug)] 58#[derive(Debug)]
@@ -57,159 +62,158 @@ struct ExpandCtx<'a> {
57 var_expanded: bool, 62 var_expanded: bool,
58} 63}
59 64
60fn expand_subtree( 65fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> {
61 template: &crate::Subtree,
62 ctx: &mut ExpandCtx,
63) -> Result<tt::Subtree, ExpandError> {
64 let mut buf: Vec<tt::TokenTree> = Vec::new(); 66 let mut buf: Vec<tt::TokenTree> = Vec::new();
65 for tt in template.token_trees.iter() { 67 for op in parse_template(template) {
66 let tt = expand_tt(tt, ctx)?; 68 match op? {
67 push_fragment(&mut buf, tt); 69 Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()),
70 Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
71 let tt = expand_subtree(ctx, tt)?;
72 buf.push(tt.into());
73 }
74 Op::Var { name, kind: _ } => {
75 let fragment = expand_var(ctx, name)?;
76 push_fragment(&mut buf, fragment);
77 }
78 Op::Repeat { subtree, kind, separator } => {
79 let fragment = expand_repeat(ctx, subtree, kind, separator)?;
80 push_fragment(&mut buf, fragment)
81 }
82 }
68 } 83 }
69
70 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) 84 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
71} 85}
72 86
73fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> { 87fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> {
74 let res: tt::TokenTree = match template { 88 let res = if v == "crate" {
75 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), 89 // FIXME: Properly handle $crate token
76 crate::TokenTree::Repeat(repeat) => { 90 let tt =
77 let mut buf: Vec<tt::TokenTree> = Vec::new(); 91 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
78 ctx.nesting.push(0); 92 .into();
79 // Dirty hack to make macro-expansion terminate. 93 Fragment::Tokens(tt)
80 // This should be replaced by a propper macro-by-example implementation 94 } else if !ctx.bindings.contains(v) {
81 let mut limit = 65536; 95 // Note that it is possible to have a `$var` inside a macro which is not bound.
82 let mut has_seps = 0; 96 // For example:
83 let mut counter = 0; 97 // ```
84 98 // macro_rules! foo {
85 // We store the old var expanded value, and restore it later 99 // ($a:ident, $b:ident, $c:tt) => {
86 // It is because before this `$repeat`, 100 // macro_rules! bar {
87 // it is possible some variables already expanad in the same subtree 101 // ($bi:ident) => {
88 // 102 // fn $bi() -> u8 {$c}
89 // `some_var_expanded` keep check if the deeper subtree has expanded variables 103 // }
90 let mut some_var_expanded = false; 104 // }
91 let old_var_expanded = ctx.var_expanded; 105 // }
92 ctx.var_expanded = false; 106 // ```
93 107 // We just treat it a normal tokens
94 while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { 108 let tt = tt::Subtree {
95 // if no var expanded in the child, we count it as a fail 109 delimiter: tt::Delimiter::None,
96 if !ctx.var_expanded { 110 token_trees: vec![
97 break; 111 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(),
98 } 112 tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
113 .into(),
114 ],
115 }
116 .into();
117 Fragment::Tokens(tt)
118 } else {
119 let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone();
120 ctx.var_expanded = true;
121 fragment
122 };
123 Ok(res)
124}
99 125
100 // Reset `ctx.var_expandeded` to see if there is other expanded variable 126fn expand_repeat(
101 // in the next matching 127 ctx: &mut ExpandCtx,
102 some_var_expanded = true; 128 template: &tt::Subtree,
103 ctx.var_expanded = false; 129 kind: RepeatKind,
104 130 separator: Option<Separator>,
105 counter += 1; 131) -> Result<Fragment, ExpandError> {
106 limit -= 1; 132 let mut buf: Vec<tt::TokenTree> = Vec::new();
107 if limit == 0 { 133 ctx.nesting.push(0);
108 log::warn!( 134 // Dirty hack to make macro-expansion terminate.
109 "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", 135 // This should be replaced by a propper macro-by-example implementation
110 template, 136 let mut limit = 65536;
111 ctx 137 let mut has_seps = 0;
112 ); 138 let mut counter = 0;
113 break; 139
114 } 140 // We store the old var expanded value, and restore it later
141 // It is because before this `$repeat`,
142 // it is possible some variables already expanad in the same subtree
143 //
144 // `some_var_expanded` keep check if the deeper subtree has expanded variables
145 let mut some_var_expanded = false;
146 let old_var_expanded = ctx.var_expanded;
147 ctx.var_expanded = false;
148
149 while let Ok(mut t) = expand_subtree(ctx, template) {
150 t.delimiter = tt::Delimiter::None;
151 // if no var expanded in the child, we count it as a fail
152 if !ctx.var_expanded {
153 break;
154 }
115 155
116 let idx = ctx.nesting.pop().unwrap(); 156 // Reset `ctx.var_expandeded` to see if there is other expanded variable
117 ctx.nesting.push(idx + 1); 157 // in the next matching
118 push_subtree(&mut buf, t); 158 some_var_expanded = true;
119 159 ctx.var_expanded = false;
120 if let Some(ref sep) = repeat.separator { 160
121 match sep { 161 counter += 1;
122 crate::Separator::Ident(ident) => { 162 limit -= 1;
123 has_seps = 1; 163 if limit == 0 {
124 buf.push(tt::Leaf::from(ident.clone()).into()); 164 log::warn!(
125 } 165 "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
126 crate::Separator::Literal(lit) => { 166 template,
127 has_seps = 1; 167 ctx
128 buf.push(tt::Leaf::from(lit.clone()).into()); 168 );
129 } 169 break;
130 170 }
131 crate::Separator::Puncts(puncts) => { 171
132 has_seps = puncts.len(); 172 let idx = ctx.nesting.pop().unwrap();
133 for punct in puncts { 173 ctx.nesting.push(idx + 1);
134 buf.push(tt::Leaf::from(*punct).into()); 174 push_subtree(&mut buf, t);
135 } 175
136 } 176 if let Some(ref sep) = separator {
137 } 177 match sep {
178 Separator::Ident(ident) => {
179 has_seps = 1;
180 buf.push(tt::Leaf::from(ident.clone()).into());
181 }
182 Separator::Literal(lit) => {
183 has_seps = 1;
184 buf.push(tt::Leaf::from(lit.clone()).into());
138 } 185 }
139 186
140 if let crate::RepeatKind::ZeroOrOne = repeat.kind { 187 Separator::Puncts(puncts) => {
141 break; 188 has_seps = puncts.len();
189 for punct in puncts {
190 buf.push(tt::Leaf::from(*punct).into());
191 }
142 } 192 }
143 } 193 }
194 }
144 195
145 // Restore the `var_expanded` by combining old one and the new one 196 if RepeatKind::ZeroOrOne == kind {
146 ctx.var_expanded = some_var_expanded || old_var_expanded; 197 break;
198 }
199 }
147 200
148 ctx.nesting.pop().unwrap(); 201 // Restore the `var_expanded` by combining old one and the new one
149 for _ in 0..has_seps { 202 ctx.var_expanded = some_var_expanded || old_var_expanded;
150 buf.pop();
151 }
152 203
153 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { 204 ctx.nesting.pop().unwrap();
154 return Err(ExpandError::UnexpectedToken); 205 for _ in 0..has_seps {
155 } 206 buf.pop();
207 }
156 208
157 // Check if it is a single token subtree without any delimiter 209 if RepeatKind::OneOrMore == kind && counter == 0 {
158 // e.g {Delimiter:None> ['>'] /Delimiter:None>} 210 return Err(ExpandError::UnexpectedToken);
159 tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() 211 }
160 } 212
161 crate::TokenTree::Leaf(leaf) => match leaf { 213 // Check if it is a single token subtree without any delimiter
162 crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident { 214 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
163 text: ident.text.clone(), 215 let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into();
164 id: tt::TokenId::unspecified(), 216 Ok(Fragment::Tokens(tt))
165 })
166 .into(),
167 crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(),
168 crate::Leaf::Var(v) => {
169 if v.text == "crate" {
170 // FIXME: Properly handle $crate token
171 tt::Leaf::from(tt::Ident {
172 text: "$crate".into(),
173 id: tt::TokenId::unspecified(),
174 })
175 .into()
176 } else if !ctx.bindings.contains(&v.text) {
177 // Note that it is possible to have a `$var` inside a macro which is not bound.
178 // For example:
179 // ```
180 // macro_rules! foo {
181 // ($a:ident, $b:ident, $c:tt) => {
182 // macro_rules! bar {
183 // ($bi:ident) => {
184 // fn $bi() -> u8 {$c}
185 // }
186 // }
187 // }
188 // ```
189 // We just treat it a normal tokens
190 tt::Subtree {
191 delimiter: tt::Delimiter::None,
192 token_trees: vec![
193 tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone })
194 .into(),
195 tt::Leaf::from(tt::Ident {
196 text: v.text.clone(),
197 id: tt::TokenId::unspecified(),
198 })
199 .into(),
200 ],
201 }
202 .into()
203 } else {
204 let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone();
205 ctx.var_expanded = true;
206 return Ok(fragment);
207 }
208 }
209 crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(),
210 },
211 };
212 Ok(Fragment::Tokens(res))
213} 217}
214 218
215fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { 219fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs
deleted file mode 100644
index 954b84d9d..000000000
--- a/crates/ra_mbe/src/mbe_parser.rs
+++ /dev/null
@@ -1,187 +0,0 @@
1use crate::tt_cursor::TtCursor;
2/// This module parses a raw `tt::TokenStream` into macro-by-example token
3/// stream. This is a *mostly* identify function, expect for handling of
4/// `$var:tt_kind` and `$(repeat),*` constructs.
5use crate::ParseError;
6
7pub(crate) fn parse(tt: &tt::Subtree) -> Result<crate::MacroRules, ParseError> {
8 let mut parser = TtCursor::new(tt);
9 let mut rules = Vec::new();
10 while !parser.is_eof() {
11 rules.push(parse_rule(&mut parser)?);
12 if let Err(e) = parser.expect_char(';') {
13 if !parser.is_eof() {
14 return Err(e);
15 }
16 break;
17 }
18 }
19 Ok(crate::MacroRules { rules })
20}
21
22fn parse_rule(p: &mut TtCursor) -> Result<crate::Rule, ParseError> {
23 let lhs = parse_subtree(p.eat_subtree()?, false)?;
24 p.expect_char('=')?;
25 p.expect_char('>')?;
26 let mut rhs = parse_subtree(p.eat_subtree()?, true)?;
27 rhs.delimiter = crate::Delimiter::None;
28 Ok(crate::Rule { lhs, rhs })
29}
30
31fn is_boolean_literal(lit: Option<&tt::TokenTree>) -> bool {
32 if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = lit {
33 if lit.text == "true" || lit.text == "false" {
34 return true;
35 }
36 }
37
38 false
39}
40
41fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree, ParseError> {
42 let mut token_trees = Vec::new();
43 let mut p = TtCursor::new(tt);
44 while let Some(tt) = p.eat() {
45 let child: crate::TokenTree = match tt {
46 tt::TokenTree::Leaf(leaf) => match leaf {
47 tt::Leaf::Punct(tt::Punct { char: '$', spacing }) => {
48 // mbe var can be an ident or keyword, including `true` and `false`
49 if p.at_ident().is_some() || is_boolean_literal(p.current()) {
50 crate::Leaf::from(parse_var(&mut p, transcriber)?).into()
51 } else if let Some(tt::TokenTree::Subtree(_)) = p.current() {
52 parse_repeat(&mut p, transcriber)?.into()
53 } else {
54 // Treat it as normal punct
55 crate::Leaf::from(tt::Punct { char: '$', spacing: *spacing }).into()
56 }
57 }
58 tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(),
59 tt::Leaf::Ident(tt::Ident { text, .. }) => {
60 crate::Leaf::from(crate::Ident { text: text.clone() }).into()
61 }
62 tt::Leaf::Literal(tt::Literal { text }) => {
63 crate::Leaf::from(crate::Literal { text: text.clone() }).into()
64 }
65 },
66 tt::TokenTree::Subtree(subtree) => parse_subtree(&subtree, transcriber)?.into(),
67 };
68 token_trees.push(child);
69 }
70 Ok(crate::Subtree { token_trees, delimiter: tt.delimiter })
71}
72
73fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseError> {
74 let text = {
75 if is_boolean_literal(p.current()) {
76 let lit = p.eat_literal().unwrap();
77 lit.text.clone()
78 } else {
79 let ident = p.eat_ident().unwrap();
80 ident.text.clone()
81 }
82 };
83
84 let kind = if !transcriber && p.at_char(':') {
85 p.bump();
86 if let Some(ident) = p.eat_ident() {
87 Some(ident.text.clone())
88 } else {
89 p.rev_bump();
90 None
91 }
92 } else {
93 None
94 };
95
96 Ok(crate::Var { text, kind })
97}
98
99fn mk_repeat(
100 rep: char,
101 subtree: crate::Subtree,
102 separator: Option<crate::Separator>,
103) -> Result<crate::Repeat, ParseError> {
104 let kind = match rep {
105 '*' => crate::RepeatKind::ZeroOrMore,
106 '+' => crate::RepeatKind::OneOrMore,
107 '?' => crate::RepeatKind::ZeroOrOne,
108 _ => return Err(ParseError::Expected(String::from("repeat"))),
109 };
110 Ok(crate::Repeat { subtree, kind, separator })
111}
112
113fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> {
114 let subtree = p.eat_subtree()?;
115 let mut subtree = parse_subtree(subtree, transcriber)?;
116 subtree.delimiter = crate::Delimiter::None;
117
118 if let Some(rep) = p.at_punct() {
119 match rep.char {
120 '*' | '+' | '?' => {
121 p.bump();
122 return mk_repeat(rep.char, subtree, None);
123 }
124 _ => {}
125 }
126 }
127
128 let sep = p.eat_seperator().ok_or_else(|| ParseError::Expected(String::from("separator")))?;
129 let rep = p.eat_punct().ok_or_else(|| ParseError::Expected(String::from("repeat")))?;
130
131 mk_repeat(rep.char, subtree, Some(sep))
132}
133
134#[cfg(test)]
135mod tests {
136 use ra_syntax::{ast, AstNode};
137
138 use super::*;
139 use crate::ast_to_token_tree;
140
141 #[test]
142 fn test_invalid_parse() {
143 expect_err("invalid", "subtree");
144
145 is_valid("($i:ident) => ()");
146 is_valid("($($i:ident)*) => ($_)");
147 is_valid("($($true:ident)*) => ($true)");
148 is_valid("($($false:ident)*) => ($false)");
149
150 expect_err("$i:ident => ()", "subtree");
151 expect_err("($i:ident) ()", "`=`");
152 expect_err("($($i:ident)_) => ()", "repeat");
153 }
154
155 fn expect_err(macro_body: &str, expected: &str) {
156 assert_eq!(
157 create_rules(&format_macro(macro_body)),
158 Err(ParseError::Expected(String::from(expected)))
159 );
160 }
161
162 fn is_valid(macro_body: &str) {
163 assert!(create_rules(&format_macro(macro_body)).is_ok());
164 }
165
166 fn format_macro(macro_body: &str) -> String {
167 format!(
168 "
169 macro_rules! foo {{
170 {}
171 }}
172",
173 macro_body
174 )
175 }
176
177 fn create_rules(macro_definition: &str) -> Result<crate::MacroRules, ParseError> {
178 let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap();
179 let macro_definition =
180 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
181
182 let (definition_tt, _) =
183 ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
184 parse(&definition_tt)
185 }
186
187}
diff --git a/crates/ra_mbe/src/parser.rs b/crates/ra_mbe/src/parser.rs
new file mode 100644
index 000000000..575f587cf
--- /dev/null
+++ b/crates/ra_mbe/src/parser.rs
@@ -0,0 +1,187 @@
1//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
2//! trees.
3
4use ra_syntax::SmolStr;
5use smallvec::SmallVec;
6
7use crate::{tt_iter::TtIter, ExpandError};
8
9#[derive(Debug)]
10pub(crate) enum Op<'a> {
11 Var { name: &'a SmolStr, kind: Option<&'a SmolStr> },
12 Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option<Separator> },
13 TokenTree(&'a tt::TokenTree),
14}
15
16#[derive(Clone, Debug, PartialEq, Eq)]
17pub(crate) enum RepeatKind {
18 ZeroOrMore,
19 OneOrMore,
20 ZeroOrOne,
21}
22
23#[derive(Clone, Debug, Eq)]
24pub(crate) enum Separator {
25 Literal(tt::Literal),
26 Ident(tt::Ident),
27 Puncts(SmallVec<[tt::Punct; 3]>),
28}
29
30// Note that when we compare a Separator, we just care about its textual value.
31impl PartialEq for Separator {
32 fn eq(&self, other: &Separator) -> bool {
33 use Separator::*;
34
35 match (self, other) {
36 (Ident(ref a), Ident(ref b)) => a.text == b.text,
37 (Literal(ref a), Literal(ref b)) => a.text == b.text,
38 (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
39 let a_iter = a.iter().map(|a| a.char);
40 let b_iter = b.iter().map(|b| b.char);
41 a_iter.eq(b_iter)
42 }
43 _ => false,
44 }
45 }
46}
47
48pub(crate) fn parse_template<'a>(
49 template: &'a tt::Subtree,
50) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
51 parse_inner(template, Mode::Template)
52}
53
54pub(crate) fn parse_pattern<'a>(
55 pattern: &'a tt::Subtree,
56) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
57 parse_inner(pattern, Mode::Pattern)
58}
59
60#[derive(Clone, Copy)]
61enum Mode {
62 Pattern,
63 Template,
64}
65
66fn parse_inner<'a>(
67 src: &'a tt::Subtree,
68 mode: Mode,
69) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> {
70 let mut src = TtIter::new(src);
71 std::iter::from_fn(move || {
72 let first = src.next()?;
73 Some(next_op(first, &mut src, mode))
74 })
75}
76
77macro_rules! err {
78 ($($tt:tt)*) => {
79 ExpandError::UnexpectedToken
80 };
81}
82
83macro_rules! bail {
84 ($($tt:tt)*) => {
85 return Err(err!($($tt)*))
86 };
87}
88
89fn next_op<'a>(
90 first: &'a tt::TokenTree,
91 src: &mut TtIter<'a>,
92 mode: Mode,
93) -> Result<Op<'a>, ExpandError> {
94 let res = match first {
95 tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
96 let second = src.next().ok_or_else(|| err!("bad var 1"))?;
97 match second {
98 tt::TokenTree::Subtree(subtree) => {
99 let (separator, kind) = parse_repeat(src)?;
100 Op::Repeat { subtree, separator, kind }
101 }
102 tt::TokenTree::Leaf(leaf) => match leaf {
103 tt::Leaf::Punct(..) => Err(ExpandError::UnexpectedToken)?,
104 tt::Leaf::Ident(ident) => {
105 let name = &ident.text;
106 let kind = eat_fragment_kind(src, mode)?;
107 Op::Var { name, kind }
108 }
109 tt::Leaf::Literal(lit) => {
110 if is_boolean_literal(lit) {
111 let name = &lit.text;
112 let kind = eat_fragment_kind(src, mode)?;
113 Op::Var { name, kind }
114 } else {
115 bail!("bad var 2");
116 }
117 }
118 },
119 }
120 }
121 tt => Op::TokenTree(tt),
122 };
123 Ok(res)
124}
125
126fn eat_fragment_kind<'a>(
127 src: &mut TtIter<'a>,
128 mode: Mode,
129) -> Result<Option<&'a SmolStr>, ExpandError> {
130 if let Mode::Pattern = mode {
131 src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?;
132 let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?;
133 return Ok(Some(&ident.text));
134 };
135 Ok(None)
136}
137
138fn is_boolean_literal(lit: &tt::Literal) -> bool {
139 match lit.text.as_str() {
140 "true" | "false" => true,
141 _ => false,
142 }
143}
144
145///TOOD: impl for slice iter
146fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> {
147 let mut separator = Separator::Puncts(SmallVec::new());
148 for tt in src {
149 let tt = match tt {
150 tt::TokenTree::Leaf(leaf) => leaf,
151 tt::TokenTree::Subtree(_) => Err(ExpandError::InvalidRepeat)?,
152 };
153 let has_sep = match &separator {
154 Separator::Puncts(puncts) => puncts.len() != 0,
155 _ => true,
156 };
157 match tt {
158 tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => {
159 Err(ExpandError::InvalidRepeat)?
160 }
161 tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()),
162 tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
163 tt::Leaf::Punct(punct) => {
164 let repeat_kind = match punct.char {
165 '*' => RepeatKind::ZeroOrMore,
166 '+' => RepeatKind::OneOrMore,
167 '?' => RepeatKind::ZeroOrOne,
168 _ => {
169 match &mut separator {
170 Separator::Puncts(puncts) => {
171 if puncts.len() == 3 {
172 Err(ExpandError::InvalidRepeat)?
173 }
174 puncts.push(punct.clone())
175 }
176 _ => Err(ExpandError::InvalidRepeat)?,
177 }
178 continue;
179 }
180 };
181 let separator = if has_sep { Some(separator) } else { None };
182 return Ok((separator, repeat_kind));
183 }
184 }
185 }
186 Err(ExpandError::InvalidRepeat)
187}
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
deleted file mode 100644
index 4440c69ff..000000000
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ /dev/null
@@ -1,91 +0,0 @@
1use crate::subtree_source::SubtreeTokenSource;
2
3use ra_parser::{FragmentKind, TokenSource, TreeSink};
4use ra_syntax::SyntaxKind;
5use tt::buffer::{Cursor, TokenBuffer};
6
7struct OffsetTokenSink<'a> {
8 cursor: Cursor<'a>,
9 error: bool,
10}
11
12impl<'a> OffsetTokenSink<'a> {
13 pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> {
14 if !self.cursor.is_root() {
15 return vec![];
16 }
17
18 let mut curr = begin;
19 let mut res = vec![];
20
21 while self.cursor != curr {
22 if let Some(token) = curr.token_tree() {
23 res.push(token);
24 }
25 curr = curr.bump();
26 }
27
28 res
29 }
30}
31
32impl<'a> TreeSink for OffsetTokenSink<'a> {
33 fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
34 for _ in 0..n_tokens {
35 self.cursor = self.cursor.bump_subtree();
36 }
37 }
38 fn start_node(&mut self, _kind: SyntaxKind) {}
39 fn finish_node(&mut self) {}
40 fn error(&mut self, _error: ra_parser::ParseError) {
41 self.error = true;
42 }
43}
44
45pub(crate) struct Parser<'a> {
46 subtree: &'a tt::Subtree,
47 cur_pos: &'a mut usize,
48}
49
50impl<'a> Parser<'a> {
51 pub fn new(cur_pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> {
52 Parser { cur_pos, subtree }
53 }
54
55 pub fn parse_fragment(self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> {
56 self.parse(|token_source, tree_skink| {
57 ra_parser::parse_fragment(token_source, tree_skink, fragment_kind)
58 })
59 }
60
61 fn parse<F>(self, f: F) -> Option<tt::TokenTree>
62 where
63 F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
64 {
65 let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
66 let mut src = SubtreeTokenSource::new(&buffer);
67 let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
68
69 f(&mut src, &mut sink);
70
71 let r = self.finish(buffer.begin(), &mut sink);
72 if sink.error {
73 return None;
74 }
75 r
76 }
77
78 fn finish(self, begin: Cursor, sink: &mut OffsetTokenSink) -> Option<tt::TokenTree> {
79 let res = sink.collect(begin);
80 *self.cur_pos += res.len();
81
82 match res.len() {
83 0 => None,
84 1 => Some(res[0].clone()),
85 _ => Some(tt::TokenTree::Subtree(tt::Subtree {
86 delimiter: tt::Delimiter::None,
87 token_trees: res.into_iter().cloned().collect(),
88 })),
89 }
90 }
91}
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 312fa4626..d7482c63d 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -3,6 +3,54 @@ use test_utils::assert_eq_text;
3 3
4use super::*; 4use super::*;
5 5
6mod rule_parsing {
7 use ra_syntax::{ast, AstNode};
8
9 use super::*;
10 use crate::ast_to_token_tree;
11
12 #[test]
13 fn test_valid_arms() {
14 fn check(macro_body: &str) {
15 let m = parse_macro_arm(macro_body);
16 m.unwrap();
17 }
18
19 check("($i:ident) => ()");
20 check("($($i:ident)*) => ($_)");
21 check("($($true:ident)*) => ($true)");
22 check("($($false:ident)*) => ($false)");
23 }
24
25 #[test]
26 fn test_invalid_arms() {
27 fn check(macro_body: &str, err: &str) {
28 let m = parse_macro_arm(macro_body);
29 assert_eq!(m, Err(ParseError::Expected(String::from(err))));
30 }
31
32 check("invalid", "expected subtree");
33
34 check("$i:ident => ()", "expected subtree");
35 check("($i:ident) ()", "expected `=`");
36 check("($($i:ident)_) => ()", "invalid repeat");
37
38 check("($i) => ($i)", "invalid macro definition");
39 check("($i:) => ($i)", "invalid macro definition");
40 }
41
42 fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> {
43 let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition);
44 let source_file = ast::SourceFile::parse(&macro_definition).ok().unwrap();
45 let macro_definition =
46 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
47
48 let (definition_tt, _) =
49 ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
50 crate::MacroRules::parse(&definition_tt)
51 }
52}
53
6// Good first issue (although a slightly challenging one): 54// Good first issue (although a slightly challenging one):
7// 55//
8// * Pick a random test from here 56// * Pick a random test from here
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs
deleted file mode 100644
index a69c006c7..000000000
--- a/crates/ra_mbe/src/tt_cursor.rs
+++ /dev/null
@@ -1,281 +0,0 @@
1use crate::{subtree_parser::Parser, ParseError};
2
3use ra_parser::FragmentKind;
4use smallvec::{smallvec, SmallVec};
5
6#[derive(Debug, Clone)]
7pub(crate) struct TtCursor<'a> {
8 subtree: &'a tt::Subtree,
9 pos: usize,
10}
11
12pub(crate) struct TtCursorMemento {
13 pos: usize,
14}
15
16impl<'a> TtCursor<'a> {
17 pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> {
18 TtCursor { subtree, pos: 0 }
19 }
20
21 pub(crate) fn is_eof(&self) -> bool {
22 self.pos == self.subtree.token_trees.len()
23 }
24
25 pub(crate) fn current(&self) -> Option<&'a tt::TokenTree> {
26 self.subtree.token_trees.get(self.pos)
27 }
28
29 pub(crate) fn at_punct(&self) -> Option<&'a tt::Punct> {
30 match self.current() {
31 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(it))) => Some(it),
32 _ => None,
33 }
34 }
35
36 pub(crate) fn at_char(&self, char: char) -> bool {
37 match self.at_punct() {
38 Some(tt::Punct { char: c, .. }) if *c == char => true,
39 _ => false,
40 }
41 }
42
43 pub(crate) fn at_ident(&mut self) -> Option<&'a tt::Ident> {
44 match self.current() {
45 Some(tt::TokenTree::Leaf(tt::Leaf::Ident(i))) => Some(i),
46 _ => None,
47 }
48 }
49
50 pub(crate) fn at_literal(&mut self) -> Option<&'a tt::Literal> {
51 match self.current() {
52 Some(tt::TokenTree::Leaf(tt::Leaf::Literal(i))) => Some(i),
53 _ => None,
54 }
55 }
56
57 pub(crate) fn bump(&mut self) {
58 self.pos += 1;
59 }
60 pub(crate) fn rev_bump(&mut self) {
61 self.pos -= 1;
62 }
63
64 pub(crate) fn eat(&mut self) -> Option<&'a tt::TokenTree> {
65 self.current().map(|it| {
66 self.bump();
67 it
68 })
69 }
70
71 pub(crate) fn eat_subtree(&mut self) -> Result<&'a tt::Subtree, ParseError> {
72 match self.current() {
73 Some(tt::TokenTree::Subtree(sub)) => {
74 self.bump();
75 Ok(sub)
76 }
77 _ => Err(ParseError::Expected(String::from("subtree"))),
78 }
79 }
80
81 pub(crate) fn eat_punct(&mut self) -> Option<&'a tt::Punct> {
82 self.at_punct().map(|it| {
83 self.bump();
84 it
85 })
86 }
87
88 pub(crate) fn eat_ident(&mut self) -> Option<&'a tt::Ident> {
89 self.at_ident().map(|i| {
90 self.bump();
91 i
92 })
93 }
94
95 pub(crate) fn eat_literal(&mut self) -> Option<&'a tt::Literal> {
96 self.at_literal().map(|i| {
97 self.bump();
98 i
99 })
100 }
101
102 pub(crate) fn eat_fragment(&mut self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> {
103 let parser = Parser::new(&mut self.pos, self.subtree);
104 parser.parse_fragment(fragment_kind)
105 }
106
107 pub(crate) fn eat_lifetime(&mut self) -> Option<tt::TokenTree> {
108 // check if it start from "`"
109 if let Some(ident) = self.at_ident() {
110 if ident.text.chars().next()? != '\'' {
111 return None;
112 }
113 }
114
115 self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into())
116 }
117
118 pub(crate) fn try_eat_vis(&mut self) -> Option<tt::TokenTree> {
119 // `vis` matcher is optional
120 let old_pos = self.pos;
121 let parser = Parser::new(&mut self.pos, self.subtree);
122
123 let res = parser.parse_fragment(FragmentKind::Visibility);
124 if res.is_none() {
125 self.pos = old_pos;
126 }
127 res
128 }
129
130 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> {
131 if self.at_char(char) {
132 self.bump();
133 Ok(())
134 } else {
135 Err(ParseError::Expected(format!("`{}`", char)))
136 }
137 }
138
139 fn eat_punct3(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
140 let sec = *self.eat_punct()?;
141 let third = *self.eat_punct()?;
142 Some(smallvec![p, sec, third])
143 }
144
145 fn eat_punct2(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> {
146 let sec = *self.eat_punct()?;
147 Some(smallvec![p, sec])
148 }
149
150 fn eat_multi_char_punct<'b, I>(
151 &mut self,
152 p: tt::Punct,
153 iter: &mut TokenPeek<'b, I>,
154 ) -> Option<SmallVec<[tt::Punct; 3]>>
155 where
156 I: Iterator<Item = &'b tt::TokenTree>,
157 {
158 if let Some((m, _)) = iter.current_punct3(p) {
159 if let r @ Some(_) = match m {
160 ('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => {
161 self.eat_punct3(p)
162 }
163 _ => None,
164 } {
165 return r;
166 }
167 }
168
169 if let Some((m, _)) = iter.current_punct2(p) {
170 if let r @ Some(_) = match m {
171 ('<', '=')
172 | ('>', '=')
173 | ('+', '=')
174 | ('-', '=')
175 | ('|', '=')
176 | ('&', '=')
177 | ('^', '=')
178 | ('/', '=')
179 | ('*', '=')
180 | ('%', '=')
181 | ('&', '&')
182 | ('|', '|')
183 | ('<', '<')
184 | ('>', '>')
185 | ('-', '>')
186 | ('!', '=')
187 | ('=', '>')
188 | ('=', '=')
189 | ('.', '.')
190 | (':', ':') => self.eat_punct2(p),
191
192 _ => None,
193 } {
194 return r;
195 }
196 }
197
198 None
199 }
200
201 pub(crate) fn eat_seperator(&mut self) -> Option<crate::Separator> {
202 match self.eat()? {
203 tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
204 Some(crate::Separator::Literal(lit.clone()))
205 }
206 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
207 Some(crate::Separator::Ident(ident.clone()))
208 }
209 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
210 match punct.char {
211 '*' | '+' | '?' => return None,
212 _ => {}
213 };
214
215 // FIXME: The parser is only handle some compositeable punct,
216 // But at this phase, some punct still is jointed.
217 // So we by pass that check here.
218 let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter());
219 let puncts = self.eat_multi_char_punct(*punct, &mut peekable);
220 let puncts = puncts.unwrap_or_else(|| smallvec![*punct]);
221
222 Some(crate::Separator::Puncts(puncts))
223 }
224 _ => None,
225 }
226 }
227
228 #[must_use]
229 pub(crate) fn save(&self) -> TtCursorMemento {
230 TtCursorMemento { pos: self.pos }
231 }
232
233 pub(crate) fn rollback(&mut self, memento: TtCursorMemento) {
234 self.pos = memento.pos;
235 }
236}
237
238pub(crate) struct TokenPeek<'a, I>
239where
240 I: Iterator<Item = &'a tt::TokenTree>,
241{
242 iter: itertools::MultiPeek<I>,
243}
244
245// helper function
246fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
247 if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
248 return Some(pp);
249 }
250 None
251}
252
253impl<'a, I> TokenPeek<'a, I>
254where
255 I: Iterator<Item = &'a tt::TokenTree>,
256{
257 pub fn new(iter: I) -> Self {
258 TokenPeek { iter: itertools::multipeek(iter) }
259 }
260
261 pub fn current_punct2(&mut self, p: tt::Punct) -> Option<((char, char), bool)> {
262 if p.spacing != tt::Spacing::Joint {
263 return None;
264 }
265
266 self.iter.reset_peek();
267 let p1 = to_punct(self.iter.peek()?)?;
268 Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
269 }
270
271 pub fn current_punct3(&mut self, p: tt::Punct) -> Option<((char, char, char), bool)> {
272 self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
273 if !last_joint {
274 None
275 } else {
276 let p2 = to_punct(*self.iter.peek()?)?;
277 Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
278 }
279 })
280 }
281}
diff --git a/crates/ra_mbe/src/tt_iter.rs b/crates/ra_mbe/src/tt_iter.rs
new file mode 100644
index 000000000..c53f99d1e
--- /dev/null
+++ b/crates/ra_mbe/src/tt_iter.rs
@@ -0,0 +1,67 @@
1#[derive(Debug, Clone)]
2pub(crate) struct TtIter<'a> {
3 pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
4}
5
6impl<'a> TtIter<'a> {
7 pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
8 TtIter { inner: subtree.token_trees.iter() }
9 }
10
11 pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> {
12 match self.next() {
13 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => {
14 Ok(())
15 }
16 _ => Err(()),
17 }
18 }
19
20 pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
21 match self.next() {
22 Some(tt::TokenTree::Subtree(it)) => Ok(it),
23 _ => Err(()),
24 }
25 }
26
27 pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
28 match self.next() {
29 Some(tt::TokenTree::Leaf(it)) => Ok(it),
30 _ => Err(()),
31 }
32 }
33
34 pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
35 match self.expect_leaf()? {
36 tt::Leaf::Ident(it) => Ok(it),
37 _ => Err(()),
38 }
39 }
40
41 pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Literal, ()> {
42 match self.expect_leaf()? {
43 tt::Leaf::Literal(it) => Ok(it),
44 _ => Err(()),
45 }
46 }
47
48 pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> {
49 match self.expect_leaf()? {
50 tt::Leaf::Punct(it) => Ok(it),
51 _ => Err(()),
52 }
53 }
54}
55
56impl<'a> Iterator for TtIter<'a> {
57 type Item = &'a tt::TokenTree;
58 fn next(&mut self) -> Option<Self::Item> {
59 self.inner.next()
60 }
61
62 fn size_hint(&self) -> (usize, Option<usize>) {
63 self.inner.size_hint()
64 }
65}
66
67impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 08:55:52 +0000