diff options
author | Aleksey Kladov <[email protected]> | 2019-09-17 00:54:22 +0100 |
---|---|---|
committer | Aleksey Kladov <[email protected]> | 2019-09-17 13:51:48 +0100 |
commit | 4551182f94fe81c314f79ddf8916a5520cfd03b0 (patch) | |
tree | e6c7b46cabe1f10f7da28f3db209df2260045fa8 /crates/ra_mbe | |
parent | 37ef8927c373b8eadd63edc1f70055428c49290e (diff) |
use usual token tree for macro expansion
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r-- | crates/ra_mbe/src/lib.rs | 157 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_expander.rs | 19 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_expander/matcher.rs | 361 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_expander/transcriber.rs | 288 | ||||
-rw-r--r-- | crates/ra_mbe/src/mbe_parser.rs | 187 | ||||
-rw-r--r-- | crates/ra_mbe/src/parser.rs | 187 | ||||
-rw-r--r-- | crates/ra_mbe/src/subtree_parser.rs | 91 | ||||
-rw-r--r-- | crates/ra_mbe/src/tests.rs | 48 | ||||
-rw-r--r-- | crates/ra_mbe/src/tt_cursor.rs | 281 | ||||
-rw-r--r-- | crates/ra_mbe/src/tt_iter.rs | 67 |
10 files changed, 753 insertions, 933 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs index f07f000ff..41720df79 100644 --- a/crates/ra_mbe/src/lib.rs +++ b/crates/ra_mbe/src/lib.rs | |||
@@ -3,30 +3,19 @@ | |||
3 | /// interface, although it contains some code to bridge `SyntaxNode`s and | 3 | /// interface, although it contains some code to bridge `SyntaxNode`s and |
4 | /// `TokenTree`s as well! | 4 | /// `TokenTree`s as well! |
5 | 5 | ||
6 | macro_rules! impl_froms { | 6 | mod parser; |
7 | ($e:ident: $($v:ident), *) => { | ||
8 | $( | ||
9 | impl From<$v> for $e { | ||
10 | fn from(it: $v) -> $e { | ||
11 | $e::$v(it) | ||
12 | } | ||
13 | } | ||
14 | )* | ||
15 | } | ||
16 | } | ||
17 | |||
18 | mod mbe_parser; | ||
19 | mod mbe_expander; | 7 | mod mbe_expander; |
20 | mod syntax_bridge; | 8 | mod syntax_bridge; |
21 | mod tt_cursor; | 9 | mod tt_iter; |
22 | mod subtree_source; | 10 | mod subtree_source; |
23 | mod subtree_parser; | ||
24 | |||
25 | use ra_syntax::SmolStr; | ||
26 | use smallvec::SmallVec; | ||
27 | 11 | ||
28 | pub use tt::{Delimiter, Punct}; | 12 | pub use tt::{Delimiter, Punct}; |
29 | 13 | ||
14 | use crate::{ | ||
15 | parser::{parse_pattern, Op}, | ||
16 | tt_iter::TtIter, | ||
17 | }; | ||
18 | |||
30 | #[derive(Debug, PartialEq, Eq)] | 19 | #[derive(Debug, PartialEq, Eq)] |
31 | pub enum ParseError { | 20 | pub enum ParseError { |
32 | Expected(String), | 21 | Expected(String), |
@@ -38,6 +27,7 @@ pub enum ExpandError { | |||
38 | UnexpectedToken, | 27 | UnexpectedToken, |
39 | BindingError(String), | 28 | BindingError(String), |
40 | ConversionError, | 29 | ConversionError, |
30 | InvalidRepeat, | ||
41 | } | 31 | } |
42 | 32 | ||
43 | pub use crate::syntax_bridge::{ | 33 | pub use crate::syntax_bridge::{ |
@@ -54,97 +44,72 @@ pub struct MacroRules { | |||
54 | pub(crate) rules: Vec<Rule>, | 44 | pub(crate) rules: Vec<Rule>, |
55 | } | 45 | } |
56 | 46 | ||
47 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
48 | pub(crate) struct Rule { | ||
49 | pub(crate) lhs: tt::Subtree, | ||
50 | pub(crate) rhs: tt::Subtree, | ||
51 | } | ||
52 | |||
57 | impl MacroRules { | 53 | impl MacroRules { |
58 | pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> { | 54 | pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> { |
59 | mbe_parser::parse(tt) | 55 | let mut src = TtIter::new(tt); |
56 | let mut rules = Vec::new(); | ||
57 | while src.len() > 0 { | ||
58 | let rule = Rule::parse(&mut src)?; | ||
59 | rules.push(rule); | ||
60 | if let Err(()) = src.expect_char(';') { | ||
61 | if src.len() > 0 { | ||
62 | return Err(ParseError::Expected("expected `:`".to_string())); | ||
63 | } | ||
64 | break; | ||
65 | } | ||
66 | } | ||
67 | Ok(MacroRules { rules }) | ||
60 | } | 68 | } |
61 | pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { | 69 | pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { |
62 | mbe_expander::expand(self, tt) | 70 | mbe_expander::expand(self, tt) |
63 | } | 71 | } |
64 | } | 72 | } |
65 | 73 | ||
66 | #[derive(Clone, Debug, PartialEq, Eq)] | 74 | impl Rule { |
67 | pub(crate) struct Rule { | 75 | fn parse(src: &mut TtIter) -> Result<Rule, ParseError> { |
68 | pub(crate) lhs: Subtree, | 76 | let mut lhs = src |
69 | pub(crate) rhs: Subtree, | 77 | .expect_subtree() |
70 | } | 78 | .map_err(|()| ParseError::Expected("expected subtree".to_string()))? |
71 | 79 | .clone(); | |
72 | #[derive(Clone, Debug, PartialEq, Eq)] | 80 | validate(&lhs)?; |
73 | pub(crate) enum TokenTree { | 81 | lhs.delimiter = tt::Delimiter::None; |
74 | Leaf(Leaf), | 82 | src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?; |
75 | Subtree(Subtree), | 83 | src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?; |
76 | Repeat(Repeat), | 84 | let mut rhs = src |
77 | } | 85 | .expect_subtree() |
78 | impl_froms!(TokenTree: Leaf, Subtree, Repeat); | 86 | .map_err(|()| ParseError::Expected("expected subtree".to_string()))? |
79 | 87 | .clone(); | |
80 | #[derive(Clone, Debug, PartialEq, Eq)] | 88 | rhs.delimiter = tt::Delimiter::None; |
81 | pub(crate) enum Leaf { | 89 | Ok(crate::Rule { lhs, rhs }) |
82 | Literal(Literal), | 90 | } |
83 | Punct(Punct), | ||
84 | Ident(Ident), | ||
85 | Var(Var), | ||
86 | } | ||
87 | impl_froms!(Leaf: Literal, Punct, Ident, Var); | ||
88 | |||
89 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
90 | pub(crate) struct Subtree { | ||
91 | pub(crate) delimiter: Delimiter, | ||
92 | pub(crate) token_trees: Vec<TokenTree>, | ||
93 | } | ||
94 | |||
95 | #[derive(Clone, Debug, Eq)] | ||
96 | pub(crate) enum Separator { | ||
97 | Literal(tt::Literal), | ||
98 | Ident(tt::Ident), | ||
99 | Puncts(SmallVec<[tt::Punct; 3]>), | ||
100 | } | 91 | } |
101 | 92 | ||
102 | // Note that when we compare a Separator, we just care about its textual value. | 93 | fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> { |
103 | impl PartialEq for crate::Separator { | 94 | for op in parse_pattern(pattern) { |
104 | fn eq(&self, other: &crate::Separator) -> bool { | 95 | let op = match op { |
105 | use crate::Separator::*; | 96 | Ok(it) => it, |
106 | 97 | Err(e) => { | |
107 | match (self, other) { | 98 | let msg = match e { |
108 | (Ident(ref a), Ident(ref b)) => a.text == b.text, | 99 | ExpandError::InvalidRepeat => "invalid repeat".to_string(), |
109 | (Literal(ref a), Literal(ref b)) => a.text == b.text, | 100 | _ => "invalid macro definition".to_string(), |
110 | (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { | 101 | }; |
111 | let a_iter = a.iter().map(|a| a.char); | 102 | return Err(ParseError::Expected(msg)); |
112 | let b_iter = b.iter().map(|b| b.char); | 103 | } |
113 | a_iter.eq(b_iter) | 104 | }; |
105 | match op { | ||
106 | Op::TokenTree(tt::TokenTree::Subtree(subtree)) | Op::Repeat { subtree, .. } => { | ||
107 | validate(subtree)? | ||
114 | } | 108 | } |
115 | _ => false, | 109 | _ => (), |
116 | } | 110 | } |
117 | } | 111 | } |
118 | } | 112 | Ok(()) |
119 | |||
120 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
121 | pub(crate) struct Repeat { | ||
122 | pub(crate) subtree: Subtree, | ||
123 | pub(crate) kind: RepeatKind, | ||
124 | pub(crate) separator: Option<Separator>, | ||
125 | } | ||
126 | |||
127 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
128 | pub(crate) enum RepeatKind { | ||
129 | ZeroOrMore, | ||
130 | OneOrMore, | ||
131 | ZeroOrOne, | ||
132 | } | ||
133 | |||
134 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
135 | pub(crate) struct Literal { | ||
136 | pub(crate) text: SmolStr, | ||
137 | } | ||
138 | |||
139 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
140 | pub(crate) struct Ident { | ||
141 | pub(crate) text: SmolStr, | ||
142 | } | ||
143 | |||
144 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
145 | pub(crate) struct Var { | ||
146 | pub(crate) text: SmolStr, | ||
147 | pub(crate) kind: Option<SmolStr>, | ||
148 | } | 113 | } |
149 | 114 | ||
150 | #[cfg(test)] | 115 | #[cfg(test)] |
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs index 15d9d83e2..b455b7321 100644 --- a/crates/ra_mbe/src/mbe_expander.rs +++ b/crates/ra_mbe/src/mbe_expander.rs | |||
@@ -8,7 +8,6 @@ mod transcriber; | |||
8 | use ra_syntax::SmolStr; | 8 | use ra_syntax::SmolStr; |
9 | use rustc_hash::FxHashMap; | 9 | use rustc_hash::FxHashMap; |
10 | 10 | ||
11 | use crate::tt_cursor::TtCursor; | ||
12 | use crate::ExpandError; | 11 | use crate::ExpandError; |
13 | 12 | ||
14 | pub(crate) fn expand( | 13 | pub(crate) fn expand( |
@@ -19,12 +18,8 @@ pub(crate) fn expand( | |||
19 | } | 18 | } |
20 | 19 | ||
21 | fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { | 20 | fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { |
22 | let mut input = TtCursor::new(input); | 21 | let bindings = matcher::match_(&rule.lhs, input)?; |
23 | let bindings = matcher::match_lhs(&rule.lhs, &mut input)?; | 22 | let res = transcriber::transcribe(&rule.rhs, &bindings)?; |
24 | if !input.is_eof() { | ||
25 | return Err(ExpandError::UnexpectedToken); | ||
26 | } | ||
27 | let res = transcriber::transcribe(&bindings, &rule.rhs)?; | ||
28 | Ok(res) | 23 | Ok(res) |
29 | } | 24 | } |
30 | 25 | ||
@@ -103,13 +98,6 @@ mod tests { | |||
103 | 98 | ||
104 | #[test] | 99 | #[test] |
105 | fn test_expand_rule() { | 100 | fn test_expand_rule() { |
106 | // FIXME: The missing $var check should be in parsing phase | ||
107 | // assert_err( | ||
108 | // "($i:ident) => ($j)", | ||
109 | // "foo!{a}", | ||
110 | // ExpandError::BindingError(String::from("could not find binding `j`")), | ||
111 | // ); | ||
112 | |||
113 | assert_err( | 101 | assert_err( |
114 | "($($i:ident);*) => ($i)", | 102 | "($($i:ident);*) => ($i)", |
115 | "foo!{a}", | 103 | "foo!{a}", |
@@ -118,9 +106,6 @@ mod tests { | |||
118 | )), | 106 | )), |
119 | ); | 107 | ); |
120 | 108 | ||
121 | assert_err("($i) => ($i)", "foo!{a}", ExpandError::UnexpectedToken); | ||
122 | assert_err("($i:) => ($i)", "foo!{a}", ExpandError::UnexpectedToken); | ||
123 | |||
124 | // FIXME: | 109 | // FIXME: |
125 | // Add an err test case for ($($i:ident)) => ($()) | 110 | // Add an err test case for ($($i:ident)) => ($()) |
126 | } | 111 | } |
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs index 100a3b0e0..aff953102 100644 --- a/crates/ra_mbe/src/mbe_expander/matcher.rs +++ b/crates/ra_mbe/src/mbe_expander/matcher.rs | |||
@@ -1,11 +1,14 @@ | |||
1 | use crate::{ | 1 | use crate::{ |
2 | mbe_expander::{Binding, Bindings, Fragment}, | 2 | mbe_expander::{Binding, Bindings, Fragment}, |
3 | tt_cursor::TtCursor, | 3 | parser::{parse_pattern, Op, RepeatKind, Separator}, |
4 | subtree_source::SubtreeTokenSource, | ||
5 | tt_iter::TtIter, | ||
4 | ExpandError, | 6 | ExpandError, |
5 | }; | 7 | }; |
6 | 8 | ||
7 | use ra_parser::FragmentKind::*; | 9 | use ra_parser::{FragmentKind::*, TreeSink}; |
8 | use ra_syntax::SmolStr; | 10 | use ra_syntax::{SmolStr, SyntaxKind}; |
11 | use tt::buffer::{Cursor, TokenBuffer}; | ||
9 | 12 | ||
10 | impl Bindings { | 13 | impl Bindings { |
11 | fn push_optional(&mut self, name: &SmolStr) { | 14 | fn push_optional(&mut self, name: &SmolStr) { |
@@ -42,121 +45,247 @@ impl Bindings { | |||
42 | } | 45 | } |
43 | Ok(()) | 46 | Ok(()) |
44 | } | 47 | } |
48 | } | ||
45 | 49 | ||
46 | fn merge(&mut self, nested: Bindings) { | 50 | macro_rules! err { |
47 | self.inner.extend(nested.inner); | 51 | () => { |
48 | } | 52 | ExpandError::BindingError(format!("")) |
53 | }; | ||
54 | ($($tt:tt)*) => { | ||
55 | ExpandError::BindingError(format!($($tt)*)) | ||
56 | }; | ||
49 | } | 57 | } |
50 | 58 | ||
51 | pub(super) fn match_lhs( | 59 | macro_rules! bail { |
52 | pattern: &crate::Subtree, | 60 | ($($tt:tt)*) => { |
53 | input: &mut TtCursor, | 61 | return Err(err!($($tt)*)) |
54 | ) -> Result<Bindings, ExpandError> { | 62 | }; |
63 | } | ||
64 | |||
65 | pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Bindings, ExpandError> { | ||
66 | assert!(pattern.delimiter == tt::Delimiter::None); | ||
67 | |||
55 | let mut res = Bindings::default(); | 68 | let mut res = Bindings::default(); |
56 | for pat in pattern.token_trees.iter() { | 69 | let mut src = TtIter::new(src); |
57 | match pat { | 70 | |
58 | crate::TokenTree::Leaf(leaf) => match leaf { | 71 | match_subtree(&mut res, pattern, &mut src)?; |
59 | crate::Leaf::Var(crate::Var { text, kind }) => { | 72 | |
60 | let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; | 73 | if src.len() > 0 { |
61 | match match_meta_var(kind.as_str(), input)? { | 74 | bail!("leftover tokens"); |
62 | Some(fragment) => { | 75 | } |
63 | res.inner.insert(text.clone(), Binding::Fragment(fragment)); | 76 | |
64 | } | 77 | Ok(res) |
65 | None => res.push_optional(text), | 78 | } |
66 | } | 79 | |
80 | fn match_subtree( | ||
81 | bindings: &mut Bindings, | ||
82 | pattern: &tt::Subtree, | ||
83 | src: &mut TtIter, | ||
84 | ) -> Result<(), ExpandError> { | ||
85 | for op in parse_pattern(pattern) { | ||
86 | match op? { | ||
87 | Op::TokenTree(tt::TokenTree::Leaf(lhs)) => { | ||
88 | let rhs = src.expect_leaf().map_err(|()| err!("expected leaf: `{}`", lhs))?; | ||
89 | match (lhs, rhs) { | ||
90 | ( | ||
91 | tt::Leaf::Punct(tt::Punct { char: lhs, .. }), | ||
92 | tt::Leaf::Punct(tt::Punct { char: rhs, .. }), | ||
93 | ) if lhs == rhs => (), | ||
94 | ( | ||
95 | tt::Leaf::Ident(tt::Ident { text: lhs, .. }), | ||
96 | tt::Leaf::Ident(tt::Ident { text: rhs, .. }), | ||
97 | ) if lhs == rhs => (), | ||
98 | ( | ||
99 | tt::Leaf::Literal(tt::Literal { text: lhs, .. }), | ||
100 | tt::Leaf::Literal(tt::Literal { text: rhs, .. }), | ||
101 | ) if lhs == rhs => (), | ||
102 | _ => Err(ExpandError::UnexpectedToken)?, | ||
67 | } | 103 | } |
68 | crate::Leaf::Punct(punct) => { | 104 | } |
69 | if !input.eat_punct().map(|p| p.char == punct.char).unwrap_or(false) { | 105 | Op::TokenTree(tt::TokenTree::Subtree(lhs)) => { |
70 | return Err(ExpandError::UnexpectedToken); | 106 | let rhs = src.expect_subtree().map_err(|()| err!("expected subtree"))?; |
71 | } | 107 | if lhs.delimiter != rhs.delimiter { |
108 | bail!("mismatched delimiter") | ||
72 | } | 109 | } |
73 | crate::Leaf::Ident(ident) => { | 110 | let mut src = TtIter::new(rhs); |
74 | if input.eat_ident().map(|i| &i.text) != Some(&ident.text) { | 111 | match_subtree(bindings, lhs, &mut src)?; |
75 | return Err(ExpandError::UnexpectedToken); | 112 | if src.len() > 0 { |
76 | } | 113 | bail!("leftover tokens"); |
77 | } | 114 | } |
78 | crate::Leaf::Literal(literal) => { | 115 | } |
79 | if input.eat_literal().map(|i| &i.text) != Some(&literal.text) { | 116 | Op::Var { name, kind } => { |
80 | return Err(ExpandError::UnexpectedToken); | 117 | let kind = kind.as_ref().ok_or(ExpandError::UnexpectedToken)?; |
118 | match match_meta_var(kind.as_str(), src)? { | ||
119 | Some(fragment) => { | ||
120 | bindings.inner.insert(name.clone(), Binding::Fragment(fragment)); | ||
81 | } | 121 | } |
122 | None => bindings.push_optional(name), | ||
82 | } | 123 | } |
124 | () | ||
125 | } | ||
126 | Op::Repeat { subtree, kind, separator } => { | ||
127 | match_repeat(bindings, subtree, kind, separator, src)? | ||
128 | } | ||
129 | } | ||
130 | } | ||
131 | Ok(()) | ||
132 | } | ||
133 | |||
134 | impl<'a> TtIter<'a> { | ||
135 | fn eat_separator(&mut self, separator: &Separator) -> bool { | ||
136 | let mut fork = self.clone(); | ||
137 | let ok = match separator { | ||
138 | Separator::Ident(lhs) => match fork.expect_ident() { | ||
139 | Ok(rhs) => rhs.text == lhs.text, | ||
140 | _ => false, | ||
83 | }, | 141 | }, |
84 | crate::TokenTree::Repeat(crate::Repeat { subtree, kind, separator }) => { | 142 | Separator::Literal(lhs) => match fork.expect_literal() { |
85 | // Dirty hack to make macro-expansion terminate. | 143 | Ok(rhs) => rhs.text == lhs.text, |
86 | // This should be replaced by a propper macro-by-example implementation | 144 | _ => false, |
87 | let mut limit = 65536; | 145 | }, |
88 | let mut counter = 0; | 146 | Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() { |
89 | 147 | Ok(rhs) => rhs.char == lhs.char, | |
90 | let mut memento = input.save(); | 148 | _ => false, |
91 | 149 | }), | |
92 | loop { | 150 | }; |
93 | match match_lhs(subtree, input) { | 151 | if ok { |
94 | Ok(nested) => { | 152 | *self = fork; |
95 | limit -= 1; | 153 | } |
96 | if limit == 0 { | 154 | ok |
97 | log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", subtree, input, kind, separator); | 155 | } |
98 | break; | ||
99 | } | ||
100 | |||
101 | memento = input.save(); | ||
102 | res.push_nested(counter, nested)?; | ||
103 | counter += 1; | ||
104 | if counter == 1 { | ||
105 | if let crate::RepeatKind::ZeroOrOne = kind { | ||
106 | break; | ||
107 | } | ||
108 | } | ||
109 | |||
110 | if let Some(separator) = separator { | ||
111 | if !input | ||
112 | .eat_seperator() | ||
113 | .map(|sep| sep == *separator) | ||
114 | .unwrap_or(false) | ||
115 | { | ||
116 | input.rollback(memento); | ||
117 | break; | ||
118 | } | ||
119 | } | ||
120 | } | ||
121 | Err(_) => { | ||
122 | input.rollback(memento); | ||
123 | break; | ||
124 | } | ||
125 | } | ||
126 | } | ||
127 | 156 | ||
128 | match kind { | 157 | pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> { |
129 | crate::RepeatKind::OneOrMore if counter == 0 => { | 158 | let ident = self.expect_ident()?; |
130 | return Err(ExpandError::UnexpectedToken); | 159 | // check if it start from "`" |
131 | } | 160 | if ident.text.chars().next() != Some('\'') { |
132 | _ if counter == 0 => { | 161 | return Err(()); |
133 | // Collect all empty variables in subtrees | 162 | } |
134 | collect_vars(subtree).iter().for_each(|s| res.push_empty(s)); | 163 | Ok(ident) |
135 | } | 164 | } |
136 | _ => {} | 165 | |
166 | pub(crate) fn expect_fragment( | ||
167 | &mut self, | ||
168 | fragment_kind: ra_parser::FragmentKind, | ||
169 | ) -> Result<tt::TokenTree, ()> { | ||
170 | pub(crate) struct OffsetTokenSink<'a> { | ||
171 | pub(crate) cursor: Cursor<'a>, | ||
172 | pub(crate) error: bool, | ||
173 | } | ||
174 | |||
175 | impl<'a> TreeSink for OffsetTokenSink<'a> { | ||
176 | fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { | ||
177 | for _ in 0..n_tokens { | ||
178 | self.cursor = self.cursor.bump_subtree(); | ||
137 | } | 179 | } |
138 | } | 180 | } |
139 | crate::TokenTree::Subtree(subtree) => { | 181 | fn start_node(&mut self, _kind: SyntaxKind) {} |
140 | let input_subtree = | 182 | fn finish_node(&mut self) {} |
141 | input.eat_subtree().map_err(|_| ExpandError::UnexpectedToken)?; | 183 | fn error(&mut self, _error: ra_parser::ParseError) { |
142 | if subtree.delimiter != input_subtree.delimiter { | 184 | self.error = true; |
143 | return Err(ExpandError::UnexpectedToken); | 185 | } |
186 | } | ||
187 | |||
188 | let buffer = TokenBuffer::new(self.inner.as_slice()); | ||
189 | let mut src = SubtreeTokenSource::new(&buffer); | ||
190 | let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; | ||
191 | |||
192 | ra_parser::parse_fragment(&mut src, &mut sink, fragment_kind); | ||
193 | |||
194 | if !sink.cursor.is_root() || sink.error { | ||
195 | return Err(()); | ||
196 | } | ||
197 | |||
198 | let mut curr = buffer.begin(); | ||
199 | let mut res = vec![]; | ||
200 | |||
201 | while curr != sink.cursor { | ||
202 | if let Some(token) = curr.token_tree() { | ||
203 | res.push(token); | ||
204 | } | ||
205 | curr = curr.bump(); | ||
206 | } | ||
207 | self.inner = self.inner.as_slice()[res.len()..].iter(); | ||
208 | match res.len() { | ||
209 | 0 => Err(()), | ||
210 | 1 => Ok(res[0].clone()), | ||
211 | _ => Ok(tt::TokenTree::Subtree(tt::Subtree { | ||
212 | delimiter: tt::Delimiter::None, | ||
213 | token_trees: res.into_iter().cloned().collect(), | ||
214 | })), | ||
215 | } | ||
216 | } | ||
217 | |||
218 | pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> { | ||
219 | let mut fork = self.clone(); | ||
220 | match fork.expect_fragment(Visibility) { | ||
221 | Ok(tt) => { | ||
222 | *self = fork; | ||
223 | Some(tt) | ||
224 | } | ||
225 | Err(()) => None, | ||
226 | } | ||
227 | } | ||
228 | } | ||
229 | |||
230 | pub(super) fn match_repeat( | ||
231 | bindings: &mut Bindings, | ||
232 | pattern: &tt::Subtree, | ||
233 | kind: RepeatKind, | ||
234 | separator: Option<Separator>, | ||
235 | src: &mut TtIter, | ||
236 | ) -> Result<(), ExpandError> { | ||
237 | // Dirty hack to make macro-expansion terminate. | ||
238 | // This should be replaced by a propper macro-by-example implementation | ||
239 | let mut limit = 65536; | ||
240 | let mut counter = 0; | ||
241 | |||
242 | for i in 0.. { | ||
243 | let mut fork = src.clone(); | ||
244 | |||
245 | if let Some(separator) = &separator { | ||
246 | if i != 0 && !fork.eat_separator(separator) { | ||
247 | break; | ||
248 | } | ||
249 | } | ||
250 | |||
251 | let mut nested = Bindings::default(); | ||
252 | match match_subtree(&mut nested, pattern, &mut fork) { | ||
253 | Ok(()) => { | ||
254 | limit -= 1; | ||
255 | if limit == 0 { | ||
256 | log::warn!("match_lhs excced in repeat pattern exceed limit => {:#?}\n{:#?}\n{:#?}\n{:#?}", pattern, src, kind, separator); | ||
257 | break; | ||
144 | } | 258 | } |
259 | *src = fork; | ||
145 | 260 | ||
146 | let mut input = TtCursor::new(input_subtree); | 261 | bindings.push_nested(counter, nested)?; |
147 | let bindings = match_lhs(&subtree, &mut input)?; | 262 | counter += 1; |
148 | if !input.is_eof() { | 263 | if counter == 1 { |
149 | return Err(ExpandError::UnexpectedToken); | 264 | if let RepeatKind::ZeroOrOne = kind { |
265 | break; | ||
266 | } | ||
150 | } | 267 | } |
268 | } | ||
269 | Err(_) => break, | ||
270 | } | ||
271 | } | ||
151 | 272 | ||
152 | res.merge(bindings); | 273 | match (kind, counter) { |
274 | (RepeatKind::OneOrMore, 0) => return Err(ExpandError::UnexpectedToken), | ||
275 | (_, 0) => { | ||
276 | // Collect all empty variables in subtrees | ||
277 | let mut vars = Vec::new(); | ||
278 | collect_vars(&mut vars, pattern)?; | ||
279 | for var in vars { | ||
280 | bindings.push_empty(&var) | ||
153 | } | 281 | } |
154 | } | 282 | } |
283 | _ => (), | ||
155 | } | 284 | } |
156 | Ok(res) | 285 | Ok(()) |
157 | } | 286 | } |
158 | 287 | ||
159 | fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, ExpandError> { | 288 | fn match_meta_var(kind: &str, input: &mut TtIter) -> Result<Option<Fragment>, ExpandError> { |
160 | let fragment = match kind { | 289 | let fragment = match kind { |
161 | "path" => Path, | 290 | "path" => Path, |
162 | "expr" => Expr, | 291 | "expr" => Expr, |
@@ -169,17 +298,20 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, | |||
169 | _ => { | 298 | _ => { |
170 | let tt = match kind { | 299 | let tt = match kind { |
171 | "ident" => { | 300 | "ident" => { |
172 | let ident = input.eat_ident().ok_or(ExpandError::UnexpectedToken)?.clone(); | 301 | let ident = input.expect_ident().map_err(|()| err!("expected ident"))?.clone(); |
173 | tt::Leaf::from(ident).into() | 302 | tt::Leaf::from(ident).into() |
174 | } | 303 | } |
175 | "tt" => input.eat().ok_or(ExpandError::UnexpectedToken)?.clone(), | 304 | "tt" => input.next().ok_or_else(|| err!())?.clone(), |
176 | "lifetime" => input.eat_lifetime().ok_or(ExpandError::UnexpectedToken)?.clone(), | 305 | "lifetime" => { |
306 | let ident = input.expect_lifetime().map_err(|()| err!())?; | ||
307 | tt::Leaf::Ident(ident.clone()).into() | ||
308 | } | ||
177 | "literal" => { | 309 | "literal" => { |
178 | let literal = input.eat_literal().ok_or(ExpandError::UnexpectedToken)?.clone(); | 310 | let literal = input.expect_literal().map_err(|()| err!())?.clone(); |
179 | tt::Leaf::from(literal).into() | 311 | tt::Leaf::from(literal).into() |
180 | } | 312 | } |
181 | // `vis` is optional | 313 | // `vis` is optional |
182 | "vis" => match input.try_eat_vis() { | 314 | "vis" => match input.eat_vis() { |
183 | Some(vis) => vis, | 315 | Some(vis) => vis, |
184 | None => return Ok(None), | 316 | None => return Ok(None), |
185 | }, | 317 | }, |
@@ -188,28 +320,19 @@ fn match_meta_var(kind: &str, input: &mut TtCursor) -> Result<Option<Fragment>, | |||
188 | return Ok(Some(Fragment::Tokens(tt))); | 320 | return Ok(Some(Fragment::Tokens(tt))); |
189 | } | 321 | } |
190 | }; | 322 | }; |
191 | let tt = input.eat_fragment(fragment).ok_or(ExpandError::UnexpectedToken)?; | 323 | let tt = input.expect_fragment(fragment).map_err(|()| err!())?; |
192 | let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; | 324 | let fragment = if kind == "expr" { Fragment::Ast(tt) } else { Fragment::Tokens(tt) }; |
193 | Ok(Some(fragment)) | 325 | Ok(Some(fragment)) |
194 | } | 326 | } |
195 | 327 | ||
196 | fn collect_vars(subtree: &crate::Subtree) -> Vec<SmolStr> { | 328 | fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> { |
197 | let mut res = Vec::new(); | 329 | for op in parse_pattern(pattern) { |
198 | 330 | match op? { | |
199 | for tkn in subtree.token_trees.iter() { | 331 | Op::Var { name, .. } => buf.push(name.clone()), |
200 | match tkn { | 332 | Op::TokenTree(tt::TokenTree::Leaf(_)) => (), |
201 | crate::TokenTree::Leaf(crate::Leaf::Var(crate::Var { text, .. })) => { | 333 | Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?, |
202 | res.push(text.clone()); | 334 | Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?, |
203 | } | ||
204 | crate::TokenTree::Subtree(subtree) => { | ||
205 | res.extend(collect_vars(subtree)); | ||
206 | } | ||
207 | crate::TokenTree::Repeat(crate::Repeat { subtree, .. }) => { | ||
208 | res.extend(collect_vars(subtree)); | ||
209 | } | ||
210 | _ => {} | ||
211 | } | 335 | } |
212 | } | 336 | } |
213 | 337 | Ok(()) | |
214 | res | ||
215 | } | 338 | } |
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs index a3df1b7de..c22680b93 100644 --- a/crates/ra_mbe/src/mbe_expander/transcriber.rs +++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs | |||
@@ -1,16 +1,20 @@ | |||
1 | //! Transcraber takes a template, like `fn $ident() {}`, a set of bindings like | ||
2 | //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` | ||
3 | |||
1 | use ra_syntax::SmolStr; | 4 | use ra_syntax::SmolStr; |
2 | 5 | ||
3 | use crate::{ | 6 | use crate::{ |
4 | mbe_expander::{Binding, Bindings, Fragment}, | 7 | mbe_expander::{Binding, Bindings, Fragment}, |
8 | parser::{parse_template, Op, RepeatKind, Separator}, | ||
5 | ExpandError, | 9 | ExpandError, |
6 | }; | 10 | }; |
7 | 11 | ||
8 | impl Bindings { | 12 | impl Bindings { |
9 | fn contains(&self, name: &SmolStr) -> bool { | 13 | fn contains(&self, name: &str) -> bool { |
10 | self.inner.contains_key(name) | 14 | self.inner.contains_key(name) |
11 | } | 15 | } |
12 | 16 | ||
13 | fn get(&self, name: &SmolStr, nesting: &[usize]) -> Result<&Fragment, ExpandError> { | 17 | fn get(&self, name: &str, nesting: &[usize]) -> Result<&Fragment, ExpandError> { |
14 | let mut b = self.inner.get(name).ok_or_else(|| { | 18 | let mut b = self.inner.get(name).ok_or_else(|| { |
15 | ExpandError::BindingError(format!("could not find binding `{}`", name)) | 19 | ExpandError::BindingError(format!("could not find binding `{}`", name)) |
16 | })?; | 20 | })?; |
@@ -43,11 +47,12 @@ impl Bindings { | |||
43 | } | 47 | } |
44 | 48 | ||
45 | pub(super) fn transcribe( | 49 | pub(super) fn transcribe( |
50 | template: &tt::Subtree, | ||
46 | bindings: &Bindings, | 51 | bindings: &Bindings, |
47 | template: &crate::Subtree, | ||
48 | ) -> Result<tt::Subtree, ExpandError> { | 52 | ) -> Result<tt::Subtree, ExpandError> { |
53 | assert!(template.delimiter == tt::Delimiter::None); | ||
49 | let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; | 54 | let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new(), var_expanded: false }; |
50 | expand_subtree(template, &mut ctx) | 55 | expand_subtree(&mut ctx, template) |
51 | } | 56 | } |
52 | 57 | ||
53 | #[derive(Debug)] | 58 | #[derive(Debug)] |
@@ -57,159 +62,158 @@ struct ExpandCtx<'a> { | |||
57 | var_expanded: bool, | 62 | var_expanded: bool, |
58 | } | 63 | } |
59 | 64 | ||
60 | fn expand_subtree( | 65 | fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { |
61 | template: &crate::Subtree, | ||
62 | ctx: &mut ExpandCtx, | ||
63 | ) -> Result<tt::Subtree, ExpandError> { | ||
64 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | 66 | let mut buf: Vec<tt::TokenTree> = Vec::new(); |
65 | for tt in template.token_trees.iter() { | 67 | for op in parse_template(template) { |
66 | let tt = expand_tt(tt, ctx)?; | 68 | match op? { |
67 | push_fragment(&mut buf, tt); | 69 | Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), |
70 | Op::TokenTree(tt::TokenTree::Subtree(tt)) => { | ||
71 | let tt = expand_subtree(ctx, tt)?; | ||
72 | buf.push(tt.into()); | ||
73 | } | ||
74 | Op::Var { name, kind: _ } => { | ||
75 | let fragment = expand_var(ctx, name)?; | ||
76 | push_fragment(&mut buf, fragment); | ||
77 | } | ||
78 | Op::Repeat { subtree, kind, separator } => { | ||
79 | let fragment = expand_repeat(ctx, subtree, kind, separator)?; | ||
80 | push_fragment(&mut buf, fragment) | ||
81 | } | ||
82 | } | ||
68 | } | 83 | } |
69 | |||
70 | Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) | 84 | Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) |
71 | } | 85 | } |
72 | 86 | ||
73 | fn expand_tt(template: &crate::TokenTree, ctx: &mut ExpandCtx) -> Result<Fragment, ExpandError> { | 87 | fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> { |
74 | let res: tt::TokenTree = match template { | 88 | let res = if v == "crate" { |
75 | crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), | 89 | // FIXME: Properly handle $crate token |
76 | crate::TokenTree::Repeat(repeat) => { | 90 | let tt = |
77 | let mut buf: Vec<tt::TokenTree> = Vec::new(); | 91 | tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) |
78 | ctx.nesting.push(0); | 92 | .into(); |
79 | // Dirty hack to make macro-expansion terminate. | 93 | Fragment::Tokens(tt) |
80 | // This should be replaced by a propper macro-by-example implementation | 94 | } else if !ctx.bindings.contains(v) { |
81 | let mut limit = 65536; | 95 | // Note that it is possible to have a `$var` inside a macro which is not bound. |
82 | let mut has_seps = 0; | 96 | // For example: |
83 | let mut counter = 0; | 97 | // ``` |
84 | 98 | // macro_rules! foo { | |
85 | // We store the old var expanded value, and restore it later | 99 | // ($a:ident, $b:ident, $c:tt) => { |
86 | // It is because before this `$repeat`, | 100 | // macro_rules! bar { |
87 | // it is possible some variables already expanad in the same subtree | 101 | // ($bi:ident) => { |
88 | // | 102 | // fn $bi() -> u8 {$c} |
89 | // `some_var_expanded` keep check if the deeper subtree has expanded variables | 103 | // } |
90 | let mut some_var_expanded = false; | 104 | // } |
91 | let old_var_expanded = ctx.var_expanded; | 105 | // } |
92 | ctx.var_expanded = false; | 106 | // ``` |
93 | 107 | // We just treat it a normal tokens | |
94 | while let Ok(t) = expand_subtree(&repeat.subtree, ctx) { | 108 | let tt = tt::Subtree { |
95 | // if no var expanded in the child, we count it as a fail | 109 | delimiter: tt::Delimiter::None, |
96 | if !ctx.var_expanded { | 110 | token_trees: vec![ |
97 | break; | 111 | tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }).into(), |
98 | } | 112 | tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() }) |
113 | .into(), | ||
114 | ], | ||
115 | } | ||
116 | .into(); | ||
117 | Fragment::Tokens(tt) | ||
118 | } else { | ||
119 | let fragment = ctx.bindings.get(&v, &ctx.nesting)?.clone(); | ||
120 | ctx.var_expanded = true; | ||
121 | fragment | ||
122 | }; | ||
123 | Ok(res) | ||
124 | } | ||
99 | 125 | ||
100 | // Reset `ctx.var_expandeded` to see if there is other expanded variable | 126 | fn expand_repeat( |
101 | // in the next matching | 127 | ctx: &mut ExpandCtx, |
102 | some_var_expanded = true; | 128 | template: &tt::Subtree, |
103 | ctx.var_expanded = false; | 129 | kind: RepeatKind, |
104 | 130 | separator: Option<Separator>, | |
105 | counter += 1; | 131 | ) -> Result<Fragment, ExpandError> { |
106 | limit -= 1; | 132 | let mut buf: Vec<tt::TokenTree> = Vec::new(); |
107 | if limit == 0 { | 133 | ctx.nesting.push(0); |
108 | log::warn!( | 134 | // Dirty hack to make macro-expansion terminate. |
109 | "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", | 135 | // This should be replaced by a propper macro-by-example implementation |
110 | template, | 136 | let mut limit = 65536; |
111 | ctx | 137 | let mut has_seps = 0; |
112 | ); | 138 | let mut counter = 0; |
113 | break; | 139 | |
114 | } | 140 | // We store the old var expanded value, and restore it later |
141 | // It is because before this `$repeat`, | ||
142 | // it is possible some variables already expanad in the same subtree | ||
143 | // | ||
144 | // `some_var_expanded` keep check if the deeper subtree has expanded variables | ||
145 | let mut some_var_expanded = false; | ||
146 | let old_var_expanded = ctx.var_expanded; | ||
147 | ctx.var_expanded = false; | ||
148 | |||
149 | while let Ok(mut t) = expand_subtree(ctx, template) { | ||
150 | t.delimiter = tt::Delimiter::None; | ||
151 | // if no var expanded in the child, we count it as a fail | ||
152 | if !ctx.var_expanded { | ||
153 | break; | ||
154 | } | ||
115 | 155 | ||
116 | let idx = ctx.nesting.pop().unwrap(); | 156 | // Reset `ctx.var_expandeded` to see if there is other expanded variable |
117 | ctx.nesting.push(idx + 1); | 157 | // in the next matching |
118 | push_subtree(&mut buf, t); | 158 | some_var_expanded = true; |
119 | 159 | ctx.var_expanded = false; | |
120 | if let Some(ref sep) = repeat.separator { | 160 | |
121 | match sep { | 161 | counter += 1; |
122 | crate::Separator::Ident(ident) => { | 162 | limit -= 1; |
123 | has_seps = 1; | 163 | if limit == 0 { |
124 | buf.push(tt::Leaf::from(ident.clone()).into()); | 164 | log::warn!( |
125 | } | 165 | "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}", |
126 | crate::Separator::Literal(lit) => { | 166 | template, |
127 | has_seps = 1; | 167 | ctx |
128 | buf.push(tt::Leaf::from(lit.clone()).into()); | 168 | ); |
129 | } | 169 | break; |
130 | 170 | } | |
131 | crate::Separator::Puncts(puncts) => { | 171 | |
132 | has_seps = puncts.len(); | 172 | let idx = ctx.nesting.pop().unwrap(); |
133 | for punct in puncts { | 173 | ctx.nesting.push(idx + 1); |
134 | buf.push(tt::Leaf::from(*punct).into()); | 174 | push_subtree(&mut buf, t); |
135 | } | 175 | |
136 | } | 176 | if let Some(ref sep) = separator { |
137 | } | 177 | match sep { |
178 | Separator::Ident(ident) => { | ||
179 | has_seps = 1; | ||
180 | buf.push(tt::Leaf::from(ident.clone()).into()); | ||
181 | } | ||
182 | Separator::Literal(lit) => { | ||
183 | has_seps = 1; | ||
184 | buf.push(tt::Leaf::from(lit.clone()).into()); | ||
138 | } | 185 | } |
139 | 186 | ||
140 | if let crate::RepeatKind::ZeroOrOne = repeat.kind { | 187 | Separator::Puncts(puncts) => { |
141 | break; | 188 | has_seps = puncts.len(); |
189 | for punct in puncts { | ||
190 | buf.push(tt::Leaf::from(*punct).into()); | ||
191 | } | ||
142 | } | 192 | } |
143 | } | 193 | } |
194 | } | ||
144 | 195 | ||
145 | // Restore the `var_expanded` by combining old one and the new one | 196 | if RepeatKind::ZeroOrOne == kind { |
146 | ctx.var_expanded = some_var_expanded || old_var_expanded; | 197 | break; |
198 | } | ||
199 | } | ||
147 | 200 | ||
148 | ctx.nesting.pop().unwrap(); | 201 | // Restore the `var_expanded` by combining old one and the new one |
149 | for _ in 0..has_seps { | 202 | ctx.var_expanded = some_var_expanded || old_var_expanded; |
150 | buf.pop(); | ||
151 | } | ||
152 | 203 | ||
153 | if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { | 204 | ctx.nesting.pop().unwrap(); |
154 | return Err(ExpandError::UnexpectedToken); | 205 | for _ in 0..has_seps { |
155 | } | 206 | buf.pop(); |
207 | } | ||
156 | 208 | ||
157 | // Check if it is a single token subtree without any delimiter | 209 | if RepeatKind::OneOrMore == kind && counter == 0 { |
158 | // e.g {Delimiter:None> ['>'] /Delimiter:None>} | 210 | return Err(ExpandError::UnexpectedToken); |
159 | tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into() | 211 | } |
160 | } | 212 | |
161 | crate::TokenTree::Leaf(leaf) => match leaf { | 213 | // Check if it is a single token subtree without any delimiter |
162 | crate::Leaf::Ident(ident) => tt::Leaf::from(tt::Ident { | 214 | // e.g {Delimiter:None> ['>'] /Delimiter:None>} |
163 | text: ident.text.clone(), | 215 | let tt = tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf }.into(); |
164 | id: tt::TokenId::unspecified(), | 216 | Ok(Fragment::Tokens(tt)) |
165 | }) | ||
166 | .into(), | ||
167 | crate::Leaf::Punct(punct) => tt::Leaf::from(*punct).into(), | ||
168 | crate::Leaf::Var(v) => { | ||
169 | if v.text == "crate" { | ||
170 | // FIXME: Properly handle $crate token | ||
171 | tt::Leaf::from(tt::Ident { | ||
172 | text: "$crate".into(), | ||
173 | id: tt::TokenId::unspecified(), | ||
174 | }) | ||
175 | .into() | ||
176 | } else if !ctx.bindings.contains(&v.text) { | ||
177 | // Note that it is possible to have a `$var` inside a macro which is not bound. | ||
178 | // For example: | ||
179 | // ``` | ||
180 | // macro_rules! foo { | ||
181 | // ($a:ident, $b:ident, $c:tt) => { | ||
182 | // macro_rules! bar { | ||
183 | // ($bi:ident) => { | ||
184 | // fn $bi() -> u8 {$c} | ||
185 | // } | ||
186 | // } | ||
187 | // } | ||
188 | // ``` | ||
189 | // We just treat it a normal tokens | ||
190 | tt::Subtree { | ||
191 | delimiter: tt::Delimiter::None, | ||
192 | token_trees: vec![ | ||
193 | tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone }) | ||
194 | .into(), | ||
195 | tt::Leaf::from(tt::Ident { | ||
196 | text: v.text.clone(), | ||
197 | id: tt::TokenId::unspecified(), | ||
198 | }) | ||
199 | .into(), | ||
200 | ], | ||
201 | } | ||
202 | .into() | ||
203 | } else { | ||
204 | let fragment = ctx.bindings.get(&v.text, &ctx.nesting)?.clone(); | ||
205 | ctx.var_expanded = true; | ||
206 | return Ok(fragment); | ||
207 | } | ||
208 | } | ||
209 | crate::Leaf::Literal(l) => tt::Leaf::from(tt::Literal { text: l.text.clone() }).into(), | ||
210 | }, | ||
211 | }; | ||
212 | Ok(Fragment::Tokens(res)) | ||
213 | } | 217 | } |
214 | 218 | ||
215 | fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { | 219 | fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { |
diff --git a/crates/ra_mbe/src/mbe_parser.rs b/crates/ra_mbe/src/mbe_parser.rs deleted file mode 100644 index 954b84d9d..000000000 --- a/crates/ra_mbe/src/mbe_parser.rs +++ /dev/null | |||
@@ -1,187 +0,0 @@ | |||
1 | use crate::tt_cursor::TtCursor; | ||
2 | /// This module parses a raw `tt::TokenStream` into macro-by-example token | ||
3 | /// stream. This is a *mostly* identify function, expect for handling of | ||
4 | /// `$var:tt_kind` and `$(repeat),*` constructs. | ||
5 | use crate::ParseError; | ||
6 | |||
7 | pub(crate) fn parse(tt: &tt::Subtree) -> Result<crate::MacroRules, ParseError> { | ||
8 | let mut parser = TtCursor::new(tt); | ||
9 | let mut rules = Vec::new(); | ||
10 | while !parser.is_eof() { | ||
11 | rules.push(parse_rule(&mut parser)?); | ||
12 | if let Err(e) = parser.expect_char(';') { | ||
13 | if !parser.is_eof() { | ||
14 | return Err(e); | ||
15 | } | ||
16 | break; | ||
17 | } | ||
18 | } | ||
19 | Ok(crate::MacroRules { rules }) | ||
20 | } | ||
21 | |||
22 | fn parse_rule(p: &mut TtCursor) -> Result<crate::Rule, ParseError> { | ||
23 | let lhs = parse_subtree(p.eat_subtree()?, false)?; | ||
24 | p.expect_char('=')?; | ||
25 | p.expect_char('>')?; | ||
26 | let mut rhs = parse_subtree(p.eat_subtree()?, true)?; | ||
27 | rhs.delimiter = crate::Delimiter::None; | ||
28 | Ok(crate::Rule { lhs, rhs }) | ||
29 | } | ||
30 | |||
31 | fn is_boolean_literal(lit: Option<&tt::TokenTree>) -> bool { | ||
32 | if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) = lit { | ||
33 | if lit.text == "true" || lit.text == "false" { | ||
34 | return true; | ||
35 | } | ||
36 | } | ||
37 | |||
38 | false | ||
39 | } | ||
40 | |||
41 | fn parse_subtree(tt: &tt::Subtree, transcriber: bool) -> Result<crate::Subtree, ParseError> { | ||
42 | let mut token_trees = Vec::new(); | ||
43 | let mut p = TtCursor::new(tt); | ||
44 | while let Some(tt) = p.eat() { | ||
45 | let child: crate::TokenTree = match tt { | ||
46 | tt::TokenTree::Leaf(leaf) => match leaf { | ||
47 | tt::Leaf::Punct(tt::Punct { char: '$', spacing }) => { | ||
48 | // mbe var can be an ident or keyword, including `true` and `false` | ||
49 | if p.at_ident().is_some() || is_boolean_literal(p.current()) { | ||
50 | crate::Leaf::from(parse_var(&mut p, transcriber)?).into() | ||
51 | } else if let Some(tt::TokenTree::Subtree(_)) = p.current() { | ||
52 | parse_repeat(&mut p, transcriber)?.into() | ||
53 | } else { | ||
54 | // Treat it as normal punct | ||
55 | crate::Leaf::from(tt::Punct { char: '$', spacing: *spacing }).into() | ||
56 | } | ||
57 | } | ||
58 | tt::Leaf::Punct(punct) => crate::Leaf::from(*punct).into(), | ||
59 | tt::Leaf::Ident(tt::Ident { text, .. }) => { | ||
60 | crate::Leaf::from(crate::Ident { text: text.clone() }).into() | ||
61 | } | ||
62 | tt::Leaf::Literal(tt::Literal { text }) => { | ||
63 | crate::Leaf::from(crate::Literal { text: text.clone() }).into() | ||
64 | } | ||
65 | }, | ||
66 | tt::TokenTree::Subtree(subtree) => parse_subtree(&subtree, transcriber)?.into(), | ||
67 | }; | ||
68 | token_trees.push(child); | ||
69 | } | ||
70 | Ok(crate::Subtree { token_trees, delimiter: tt.delimiter }) | ||
71 | } | ||
72 | |||
73 | fn parse_var(p: &mut TtCursor, transcriber: bool) -> Result<crate::Var, ParseError> { | ||
74 | let text = { | ||
75 | if is_boolean_literal(p.current()) { | ||
76 | let lit = p.eat_literal().unwrap(); | ||
77 | lit.text.clone() | ||
78 | } else { | ||
79 | let ident = p.eat_ident().unwrap(); | ||
80 | ident.text.clone() | ||
81 | } | ||
82 | }; | ||
83 | |||
84 | let kind = if !transcriber && p.at_char(':') { | ||
85 | p.bump(); | ||
86 | if let Some(ident) = p.eat_ident() { | ||
87 | Some(ident.text.clone()) | ||
88 | } else { | ||
89 | p.rev_bump(); | ||
90 | None | ||
91 | } | ||
92 | } else { | ||
93 | None | ||
94 | }; | ||
95 | |||
96 | Ok(crate::Var { text, kind }) | ||
97 | } | ||
98 | |||
99 | fn mk_repeat( | ||
100 | rep: char, | ||
101 | subtree: crate::Subtree, | ||
102 | separator: Option<crate::Separator>, | ||
103 | ) -> Result<crate::Repeat, ParseError> { | ||
104 | let kind = match rep { | ||
105 | '*' => crate::RepeatKind::ZeroOrMore, | ||
106 | '+' => crate::RepeatKind::OneOrMore, | ||
107 | '?' => crate::RepeatKind::ZeroOrOne, | ||
108 | _ => return Err(ParseError::Expected(String::from("repeat"))), | ||
109 | }; | ||
110 | Ok(crate::Repeat { subtree, kind, separator }) | ||
111 | } | ||
112 | |||
113 | fn parse_repeat(p: &mut TtCursor, transcriber: bool) -> Result<crate::Repeat, ParseError> { | ||
114 | let subtree = p.eat_subtree()?; | ||
115 | let mut subtree = parse_subtree(subtree, transcriber)?; | ||
116 | subtree.delimiter = crate::Delimiter::None; | ||
117 | |||
118 | if let Some(rep) = p.at_punct() { | ||
119 | match rep.char { | ||
120 | '*' | '+' | '?' => { | ||
121 | p.bump(); | ||
122 | return mk_repeat(rep.char, subtree, None); | ||
123 | } | ||
124 | _ => {} | ||
125 | } | ||
126 | } | ||
127 | |||
128 | let sep = p.eat_seperator().ok_or_else(|| ParseError::Expected(String::from("separator")))?; | ||
129 | let rep = p.eat_punct().ok_or_else(|| ParseError::Expected(String::from("repeat")))?; | ||
130 | |||
131 | mk_repeat(rep.char, subtree, Some(sep)) | ||
132 | } | ||
133 | |||
134 | #[cfg(test)] | ||
135 | mod tests { | ||
136 | use ra_syntax::{ast, AstNode}; | ||
137 | |||
138 | use super::*; | ||
139 | use crate::ast_to_token_tree; | ||
140 | |||
141 | #[test] | ||
142 | fn test_invalid_parse() { | ||
143 | expect_err("invalid", "subtree"); | ||
144 | |||
145 | is_valid("($i:ident) => ()"); | ||
146 | is_valid("($($i:ident)*) => ($_)"); | ||
147 | is_valid("($($true:ident)*) => ($true)"); | ||
148 | is_valid("($($false:ident)*) => ($false)"); | ||
149 | |||
150 | expect_err("$i:ident => ()", "subtree"); | ||
151 | expect_err("($i:ident) ()", "`=`"); | ||
152 | expect_err("($($i:ident)_) => ()", "repeat"); | ||
153 | } | ||
154 | |||
155 | fn expect_err(macro_body: &str, expected: &str) { | ||
156 | assert_eq!( | ||
157 | create_rules(&format_macro(macro_body)), | ||
158 | Err(ParseError::Expected(String::from(expected))) | ||
159 | ); | ||
160 | } | ||
161 | |||
162 | fn is_valid(macro_body: &str) { | ||
163 | assert!(create_rules(&format_macro(macro_body)).is_ok()); | ||
164 | } | ||
165 | |||
166 | fn format_macro(macro_body: &str) -> String { | ||
167 | format!( | ||
168 | " | ||
169 | macro_rules! foo {{ | ||
170 | {} | ||
171 | }} | ||
172 | ", | ||
173 | macro_body | ||
174 | ) | ||
175 | } | ||
176 | |||
177 | fn create_rules(macro_definition: &str) -> Result<crate::MacroRules, ParseError> { | ||
178 | let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap(); | ||
179 | let macro_definition = | ||
180 | source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); | ||
181 | |||
182 | let (definition_tt, _) = | ||
183 | ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); | ||
184 | parse(&definition_tt) | ||
185 | } | ||
186 | |||
187 | } | ||
diff --git a/crates/ra_mbe/src/parser.rs b/crates/ra_mbe/src/parser.rs new file mode 100644 index 000000000..575f587cf --- /dev/null +++ b/crates/ra_mbe/src/parser.rs | |||
@@ -0,0 +1,187 @@ | |||
1 | //! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token | ||
2 | //! trees. | ||
3 | |||
4 | use ra_syntax::SmolStr; | ||
5 | use smallvec::SmallVec; | ||
6 | |||
7 | use crate::{tt_iter::TtIter, ExpandError}; | ||
8 | |||
9 | #[derive(Debug)] | ||
10 | pub(crate) enum Op<'a> { | ||
11 | Var { name: &'a SmolStr, kind: Option<&'a SmolStr> }, | ||
12 | Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option<Separator> }, | ||
13 | TokenTree(&'a tt::TokenTree), | ||
14 | } | ||
15 | |||
16 | #[derive(Clone, Debug, PartialEq, Eq)] | ||
17 | pub(crate) enum RepeatKind { | ||
18 | ZeroOrMore, | ||
19 | OneOrMore, | ||
20 | ZeroOrOne, | ||
21 | } | ||
22 | |||
23 | #[derive(Clone, Debug, Eq)] | ||
24 | pub(crate) enum Separator { | ||
25 | Literal(tt::Literal), | ||
26 | Ident(tt::Ident), | ||
27 | Puncts(SmallVec<[tt::Punct; 3]>), | ||
28 | } | ||
29 | |||
30 | // Note that when we compare a Separator, we just care about its textual value. | ||
31 | impl PartialEq for Separator { | ||
32 | fn eq(&self, other: &Separator) -> bool { | ||
33 | use Separator::*; | ||
34 | |||
35 | match (self, other) { | ||
36 | (Ident(ref a), Ident(ref b)) => a.text == b.text, | ||
37 | (Literal(ref a), Literal(ref b)) => a.text == b.text, | ||
38 | (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => { | ||
39 | let a_iter = a.iter().map(|a| a.char); | ||
40 | let b_iter = b.iter().map(|b| b.char); | ||
41 | a_iter.eq(b_iter) | ||
42 | } | ||
43 | _ => false, | ||
44 | } | ||
45 | } | ||
46 | } | ||
47 | |||
48 | pub(crate) fn parse_template<'a>( | ||
49 | template: &'a tt::Subtree, | ||
50 | ) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> { | ||
51 | parse_inner(template, Mode::Template) | ||
52 | } | ||
53 | |||
54 | pub(crate) fn parse_pattern<'a>( | ||
55 | pattern: &'a tt::Subtree, | ||
56 | ) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> { | ||
57 | parse_inner(pattern, Mode::Pattern) | ||
58 | } | ||
59 | |||
60 | #[derive(Clone, Copy)] | ||
61 | enum Mode { | ||
62 | Pattern, | ||
63 | Template, | ||
64 | } | ||
65 | |||
66 | fn parse_inner<'a>( | ||
67 | src: &'a tt::Subtree, | ||
68 | mode: Mode, | ||
69 | ) -> impl Iterator<Item = Result<Op<'a>, ExpandError>> { | ||
70 | let mut src = TtIter::new(src); | ||
71 | std::iter::from_fn(move || { | ||
72 | let first = src.next()?; | ||
73 | Some(next_op(first, &mut src, mode)) | ||
74 | }) | ||
75 | } | ||
76 | |||
77 | macro_rules! err { | ||
78 | ($($tt:tt)*) => { | ||
79 | ExpandError::UnexpectedToken | ||
80 | }; | ||
81 | } | ||
82 | |||
83 | macro_rules! bail { | ||
84 | ($($tt:tt)*) => { | ||
85 | return Err(err!($($tt)*)) | ||
86 | }; | ||
87 | } | ||
88 | |||
89 | fn next_op<'a>( | ||
90 | first: &'a tt::TokenTree, | ||
91 | src: &mut TtIter<'a>, | ||
92 | mode: Mode, | ||
93 | ) -> Result<Op<'a>, ExpandError> { | ||
94 | let res = match first { | ||
95 | tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => { | ||
96 | let second = src.next().ok_or_else(|| err!("bad var 1"))?; | ||
97 | match second { | ||
98 | tt::TokenTree::Subtree(subtree) => { | ||
99 | let (separator, kind) = parse_repeat(src)?; | ||
100 | Op::Repeat { subtree, separator, kind } | ||
101 | } | ||
102 | tt::TokenTree::Leaf(leaf) => match leaf { | ||
103 | tt::Leaf::Punct(..) => Err(ExpandError::UnexpectedToken)?, | ||
104 | tt::Leaf::Ident(ident) => { | ||
105 | let name = &ident.text; | ||
106 | let kind = eat_fragment_kind(src, mode)?; | ||
107 | Op::Var { name, kind } | ||
108 | } | ||
109 | tt::Leaf::Literal(lit) => { | ||
110 | if is_boolean_literal(lit) { | ||
111 | let name = &lit.text; | ||
112 | let kind = eat_fragment_kind(src, mode)?; | ||
113 | Op::Var { name, kind } | ||
114 | } else { | ||
115 | bail!("bad var 2"); | ||
116 | } | ||
117 | } | ||
118 | }, | ||
119 | } | ||
120 | } | ||
121 | tt => Op::TokenTree(tt), | ||
122 | }; | ||
123 | Ok(res) | ||
124 | } | ||
125 | |||
126 | fn eat_fragment_kind<'a>( | ||
127 | src: &mut TtIter<'a>, | ||
128 | mode: Mode, | ||
129 | ) -> Result<Option<&'a SmolStr>, ExpandError> { | ||
130 | if let Mode::Pattern = mode { | ||
131 | src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?; | ||
132 | let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?; | ||
133 | return Ok(Some(&ident.text)); | ||
134 | }; | ||
135 | Ok(None) | ||
136 | } | ||
137 | |||
138 | fn is_boolean_literal(lit: &tt::Literal) -> bool { | ||
139 | match lit.text.as_str() { | ||
140 | "true" | "false" => true, | ||
141 | _ => false, | ||
142 | } | ||
143 | } | ||
144 | |||
145 | ///TOOD: impl for slice iter | ||
146 | fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> { | ||
147 | let mut separator = Separator::Puncts(SmallVec::new()); | ||
148 | for tt in src { | ||
149 | let tt = match tt { | ||
150 | tt::TokenTree::Leaf(leaf) => leaf, | ||
151 | tt::TokenTree::Subtree(_) => Err(ExpandError::InvalidRepeat)?, | ||
152 | }; | ||
153 | let has_sep = match &separator { | ||
154 | Separator::Puncts(puncts) => puncts.len() != 0, | ||
155 | _ => true, | ||
156 | }; | ||
157 | match tt { | ||
158 | tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => { | ||
159 | Err(ExpandError::InvalidRepeat)? | ||
160 | } | ||
161 | tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()), | ||
162 | tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()), | ||
163 | tt::Leaf::Punct(punct) => { | ||
164 | let repeat_kind = match punct.char { | ||
165 | '*' => RepeatKind::ZeroOrMore, | ||
166 | '+' => RepeatKind::OneOrMore, | ||
167 | '?' => RepeatKind::ZeroOrOne, | ||
168 | _ => { | ||
169 | match &mut separator { | ||
170 | Separator::Puncts(puncts) => { | ||
171 | if puncts.len() == 3 { | ||
172 | Err(ExpandError::InvalidRepeat)? | ||
173 | } | ||
174 | puncts.push(punct.clone()) | ||
175 | } | ||
176 | _ => Err(ExpandError::InvalidRepeat)?, | ||
177 | } | ||
178 | continue; | ||
179 | } | ||
180 | }; | ||
181 | let separator = if has_sep { Some(separator) } else { None }; | ||
182 | return Ok((separator, repeat_kind)); | ||
183 | } | ||
184 | } | ||
185 | } | ||
186 | Err(ExpandError::InvalidRepeat) | ||
187 | } | ||
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs deleted file mode 100644 index 4440c69ff..000000000 --- a/crates/ra_mbe/src/subtree_parser.rs +++ /dev/null | |||
@@ -1,91 +0,0 @@ | |||
1 | use crate::subtree_source::SubtreeTokenSource; | ||
2 | |||
3 | use ra_parser::{FragmentKind, TokenSource, TreeSink}; | ||
4 | use ra_syntax::SyntaxKind; | ||
5 | use tt::buffer::{Cursor, TokenBuffer}; | ||
6 | |||
7 | struct OffsetTokenSink<'a> { | ||
8 | cursor: Cursor<'a>, | ||
9 | error: bool, | ||
10 | } | ||
11 | |||
12 | impl<'a> OffsetTokenSink<'a> { | ||
13 | pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> { | ||
14 | if !self.cursor.is_root() { | ||
15 | return vec![]; | ||
16 | } | ||
17 | |||
18 | let mut curr = begin; | ||
19 | let mut res = vec![]; | ||
20 | |||
21 | while self.cursor != curr { | ||
22 | if let Some(token) = curr.token_tree() { | ||
23 | res.push(token); | ||
24 | } | ||
25 | curr = curr.bump(); | ||
26 | } | ||
27 | |||
28 | res | ||
29 | } | ||
30 | } | ||
31 | |||
32 | impl<'a> TreeSink for OffsetTokenSink<'a> { | ||
33 | fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) { | ||
34 | for _ in 0..n_tokens { | ||
35 | self.cursor = self.cursor.bump_subtree(); | ||
36 | } | ||
37 | } | ||
38 | fn start_node(&mut self, _kind: SyntaxKind) {} | ||
39 | fn finish_node(&mut self) {} | ||
40 | fn error(&mut self, _error: ra_parser::ParseError) { | ||
41 | self.error = true; | ||
42 | } | ||
43 | } | ||
44 | |||
45 | pub(crate) struct Parser<'a> { | ||
46 | subtree: &'a tt::Subtree, | ||
47 | cur_pos: &'a mut usize, | ||
48 | } | ||
49 | |||
50 | impl<'a> Parser<'a> { | ||
51 | pub fn new(cur_pos: &'a mut usize, subtree: &'a tt::Subtree) -> Parser<'a> { | ||
52 | Parser { cur_pos, subtree } | ||
53 | } | ||
54 | |||
55 | pub fn parse_fragment(self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> { | ||
56 | self.parse(|token_source, tree_skink| { | ||
57 | ra_parser::parse_fragment(token_source, tree_skink, fragment_kind) | ||
58 | }) | ||
59 | } | ||
60 | |||
61 | fn parse<F>(self, f: F) -> Option<tt::TokenTree> | ||
62 | where | ||
63 | F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink), | ||
64 | { | ||
65 | let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]); | ||
66 | let mut src = SubtreeTokenSource::new(&buffer); | ||
67 | let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; | ||
68 | |||
69 | f(&mut src, &mut sink); | ||
70 | |||
71 | let r = self.finish(buffer.begin(), &mut sink); | ||
72 | if sink.error { | ||
73 | return None; | ||
74 | } | ||
75 | r | ||
76 | } | ||
77 | |||
78 | fn finish(self, begin: Cursor, sink: &mut OffsetTokenSink) -> Option<tt::TokenTree> { | ||
79 | let res = sink.collect(begin); | ||
80 | *self.cur_pos += res.len(); | ||
81 | |||
82 | match res.len() { | ||
83 | 0 => None, | ||
84 | 1 => Some(res[0].clone()), | ||
85 | _ => Some(tt::TokenTree::Subtree(tt::Subtree { | ||
86 | delimiter: tt::Delimiter::None, | ||
87 | token_trees: res.into_iter().cloned().collect(), | ||
88 | })), | ||
89 | } | ||
90 | } | ||
91 | } | ||
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs index 312fa4626..d7482c63d 100644 --- a/crates/ra_mbe/src/tests.rs +++ b/crates/ra_mbe/src/tests.rs | |||
@@ -3,6 +3,54 @@ use test_utils::assert_eq_text; | |||
3 | 3 | ||
4 | use super::*; | 4 | use super::*; |
5 | 5 | ||
6 | mod rule_parsing { | ||
7 | use ra_syntax::{ast, AstNode}; | ||
8 | |||
9 | use super::*; | ||
10 | use crate::ast_to_token_tree; | ||
11 | |||
12 | #[test] | ||
13 | fn test_valid_arms() { | ||
14 | fn check(macro_body: &str) { | ||
15 | let m = parse_macro_arm(macro_body); | ||
16 | m.unwrap(); | ||
17 | } | ||
18 | |||
19 | check("($i:ident) => ()"); | ||
20 | check("($($i:ident)*) => ($_)"); | ||
21 | check("($($true:ident)*) => ($true)"); | ||
22 | check("($($false:ident)*) => ($false)"); | ||
23 | } | ||
24 | |||
25 | #[test] | ||
26 | fn test_invalid_arms() { | ||
27 | fn check(macro_body: &str, err: &str) { | ||
28 | let m = parse_macro_arm(macro_body); | ||
29 | assert_eq!(m, Err(ParseError::Expected(String::from(err)))); | ||
30 | } | ||
31 | |||
32 | check("invalid", "expected subtree"); | ||
33 | |||
34 | check("$i:ident => ()", "expected subtree"); | ||
35 | check("($i:ident) ()", "expected `=`"); | ||
36 | check("($($i:ident)_) => ()", "invalid repeat"); | ||
37 | |||
38 | check("($i) => ($i)", "invalid macro definition"); | ||
39 | check("($i:) => ($i)", "invalid macro definition"); | ||
40 | } | ||
41 | |||
42 | fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> { | ||
43 | let macro_definition = format!(" macro_rules! m {{ {} }} ", arm_definition); | ||
44 | let source_file = ast::SourceFile::parse(¯o_definition).ok().unwrap(); | ||
45 | let macro_definition = | ||
46 | source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); | ||
47 | |||
48 | let (definition_tt, _) = | ||
49 | ast_to_token_tree(¯o_definition.token_tree().unwrap()).unwrap(); | ||
50 | crate::MacroRules::parse(&definition_tt) | ||
51 | } | ||
52 | } | ||
53 | |||
6 | // Good first issue (although a slightly challenging one): | 54 | // Good first issue (although a slightly challenging one): |
7 | // | 55 | // |
8 | // * Pick a random test from here | 56 | // * Pick a random test from here |
diff --git a/crates/ra_mbe/src/tt_cursor.rs b/crates/ra_mbe/src/tt_cursor.rs deleted file mode 100644 index a69c006c7..000000000 --- a/crates/ra_mbe/src/tt_cursor.rs +++ /dev/null | |||
@@ -1,281 +0,0 @@ | |||
1 | use crate::{subtree_parser::Parser, ParseError}; | ||
2 | |||
3 | use ra_parser::FragmentKind; | ||
4 | use smallvec::{smallvec, SmallVec}; | ||
5 | |||
6 | #[derive(Debug, Clone)] | ||
7 | pub(crate) struct TtCursor<'a> { | ||
8 | subtree: &'a tt::Subtree, | ||
9 | pos: usize, | ||
10 | } | ||
11 | |||
12 | pub(crate) struct TtCursorMemento { | ||
13 | pos: usize, | ||
14 | } | ||
15 | |||
16 | impl<'a> TtCursor<'a> { | ||
17 | pub(crate) fn new(subtree: &'a tt::Subtree) -> TtCursor<'a> { | ||
18 | TtCursor { subtree, pos: 0 } | ||
19 | } | ||
20 | |||
21 | pub(crate) fn is_eof(&self) -> bool { | ||
22 | self.pos == self.subtree.token_trees.len() | ||
23 | } | ||
24 | |||
25 | pub(crate) fn current(&self) -> Option<&'a tt::TokenTree> { | ||
26 | self.subtree.token_trees.get(self.pos) | ||
27 | } | ||
28 | |||
29 | pub(crate) fn at_punct(&self) -> Option<&'a tt::Punct> { | ||
30 | match self.current() { | ||
31 | Some(tt::TokenTree::Leaf(tt::Leaf::Punct(it))) => Some(it), | ||
32 | _ => None, | ||
33 | } | ||
34 | } | ||
35 | |||
36 | pub(crate) fn at_char(&self, char: char) -> bool { | ||
37 | match self.at_punct() { | ||
38 | Some(tt::Punct { char: c, .. }) if *c == char => true, | ||
39 | _ => false, | ||
40 | } | ||
41 | } | ||
42 | |||
43 | pub(crate) fn at_ident(&mut self) -> Option<&'a tt::Ident> { | ||
44 | match self.current() { | ||
45 | Some(tt::TokenTree::Leaf(tt::Leaf::Ident(i))) => Some(i), | ||
46 | _ => None, | ||
47 | } | ||
48 | } | ||
49 | |||
50 | pub(crate) fn at_literal(&mut self) -> Option<&'a tt::Literal> { | ||
51 | match self.current() { | ||
52 | Some(tt::TokenTree::Leaf(tt::Leaf::Literal(i))) => Some(i), | ||
53 | _ => None, | ||
54 | } | ||
55 | } | ||
56 | |||
57 | pub(crate) fn bump(&mut self) { | ||
58 | self.pos += 1; | ||
59 | } | ||
60 | pub(crate) fn rev_bump(&mut self) { | ||
61 | self.pos -= 1; | ||
62 | } | ||
63 | |||
64 | pub(crate) fn eat(&mut self) -> Option<&'a tt::TokenTree> { | ||
65 | self.current().map(|it| { | ||
66 | self.bump(); | ||
67 | it | ||
68 | }) | ||
69 | } | ||
70 | |||
71 | pub(crate) fn eat_subtree(&mut self) -> Result<&'a tt::Subtree, ParseError> { | ||
72 | match self.current() { | ||
73 | Some(tt::TokenTree::Subtree(sub)) => { | ||
74 | self.bump(); | ||
75 | Ok(sub) | ||
76 | } | ||
77 | _ => Err(ParseError::Expected(String::from("subtree"))), | ||
78 | } | ||
79 | } | ||
80 | |||
81 | pub(crate) fn eat_punct(&mut self) -> Option<&'a tt::Punct> { | ||
82 | self.at_punct().map(|it| { | ||
83 | self.bump(); | ||
84 | it | ||
85 | }) | ||
86 | } | ||
87 | |||
88 | pub(crate) fn eat_ident(&mut self) -> Option<&'a tt::Ident> { | ||
89 | self.at_ident().map(|i| { | ||
90 | self.bump(); | ||
91 | i | ||
92 | }) | ||
93 | } | ||
94 | |||
95 | pub(crate) fn eat_literal(&mut self) -> Option<&'a tt::Literal> { | ||
96 | self.at_literal().map(|i| { | ||
97 | self.bump(); | ||
98 | i | ||
99 | }) | ||
100 | } | ||
101 | |||
102 | pub(crate) fn eat_fragment(&mut self, fragment_kind: FragmentKind) -> Option<tt::TokenTree> { | ||
103 | let parser = Parser::new(&mut self.pos, self.subtree); | ||
104 | parser.parse_fragment(fragment_kind) | ||
105 | } | ||
106 | |||
107 | pub(crate) fn eat_lifetime(&mut self) -> Option<tt::TokenTree> { | ||
108 | // check if it start from "`" | ||
109 | if let Some(ident) = self.at_ident() { | ||
110 | if ident.text.chars().next()? != '\'' { | ||
111 | return None; | ||
112 | } | ||
113 | } | ||
114 | |||
115 | self.eat_ident().cloned().map(|ident| tt::Leaf::from(ident).into()) | ||
116 | } | ||
117 | |||
118 | pub(crate) fn try_eat_vis(&mut self) -> Option<tt::TokenTree> { | ||
119 | // `vis` matcher is optional | ||
120 | let old_pos = self.pos; | ||
121 | let parser = Parser::new(&mut self.pos, self.subtree); | ||
122 | |||
123 | let res = parser.parse_fragment(FragmentKind::Visibility); | ||
124 | if res.is_none() { | ||
125 | self.pos = old_pos; | ||
126 | } | ||
127 | res | ||
128 | } | ||
129 | |||
130 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> { | ||
131 | if self.at_char(char) { | ||
132 | self.bump(); | ||
133 | Ok(()) | ||
134 | } else { | ||
135 | Err(ParseError::Expected(format!("`{}`", char))) | ||
136 | } | ||
137 | } | ||
138 | |||
139 | fn eat_punct3(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> { | ||
140 | let sec = *self.eat_punct()?; | ||
141 | let third = *self.eat_punct()?; | ||
142 | Some(smallvec![p, sec, third]) | ||
143 | } | ||
144 | |||
145 | fn eat_punct2(&mut self, p: tt::Punct) -> Option<SmallVec<[tt::Punct; 3]>> { | ||
146 | let sec = *self.eat_punct()?; | ||
147 | Some(smallvec![p, sec]) | ||
148 | } | ||
149 | |||
150 | fn eat_multi_char_punct<'b, I>( | ||
151 | &mut self, | ||
152 | p: tt::Punct, | ||
153 | iter: &mut TokenPeek<'b, I>, | ||
154 | ) -> Option<SmallVec<[tt::Punct; 3]>> | ||
155 | where | ||
156 | I: Iterator<Item = &'b tt::TokenTree>, | ||
157 | { | ||
158 | if let Some((m, _)) = iter.current_punct3(p) { | ||
159 | if let r @ Some(_) = match m { | ||
160 | ('<', '<', '=') | ('>', '>', '=') | ('.', '.', '.') | ('.', '.', '=') => { | ||
161 | self.eat_punct3(p) | ||
162 | } | ||
163 | _ => None, | ||
164 | } { | ||
165 | return r; | ||
166 | } | ||
167 | } | ||
168 | |||
169 | if let Some((m, _)) = iter.current_punct2(p) { | ||
170 | if let r @ Some(_) = match m { | ||
171 | ('<', '=') | ||
172 | | ('>', '=') | ||
173 | | ('+', '=') | ||
174 | | ('-', '=') | ||
175 | | ('|', '=') | ||
176 | | ('&', '=') | ||
177 | | ('^', '=') | ||
178 | | ('/', '=') | ||
179 | | ('*', '=') | ||
180 | | ('%', '=') | ||
181 | | ('&', '&') | ||
182 | | ('|', '|') | ||
183 | | ('<', '<') | ||
184 | | ('>', '>') | ||
185 | | ('-', '>') | ||
186 | | ('!', '=') | ||
187 | | ('=', '>') | ||
188 | | ('=', '=') | ||
189 | | ('.', '.') | ||
190 | | (':', ':') => self.eat_punct2(p), | ||
191 | |||
192 | _ => None, | ||
193 | } { | ||
194 | return r; | ||
195 | } | ||
196 | } | ||
197 | |||
198 | None | ||
199 | } | ||
200 | |||
201 | pub(crate) fn eat_seperator(&mut self) -> Option<crate::Separator> { | ||
202 | match self.eat()? { | ||
203 | tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { | ||
204 | Some(crate::Separator::Literal(lit.clone())) | ||
205 | } | ||
206 | tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => { | ||
207 | Some(crate::Separator::Ident(ident.clone())) | ||
208 | } | ||
209 | tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => { | ||
210 | match punct.char { | ||
211 | '*' | '+' | '?' => return None, | ||
212 | _ => {} | ||
213 | }; | ||
214 | |||
215 | // FIXME: The parser is only handle some compositeable punct, | ||
216 | // But at this phase, some punct still is jointed. | ||
217 | // So we by pass that check here. | ||
218 | let mut peekable = TokenPeek::new(self.subtree.token_trees[self.pos..].iter()); | ||
219 | let puncts = self.eat_multi_char_punct(*punct, &mut peekable); | ||
220 | let puncts = puncts.unwrap_or_else(|| smallvec![*punct]); | ||
221 | |||
222 | Some(crate::Separator::Puncts(puncts)) | ||
223 | } | ||
224 | _ => None, | ||
225 | } | ||
226 | } | ||
227 | |||
228 | #[must_use] | ||
229 | pub(crate) fn save(&self) -> TtCursorMemento { | ||
230 | TtCursorMemento { pos: self.pos } | ||
231 | } | ||
232 | |||
233 | pub(crate) fn rollback(&mut self, memento: TtCursorMemento) { | ||
234 | self.pos = memento.pos; | ||
235 | } | ||
236 | } | ||
237 | |||
238 | pub(crate) struct TokenPeek<'a, I> | ||
239 | where | ||
240 | I: Iterator<Item = &'a tt::TokenTree>, | ||
241 | { | ||
242 | iter: itertools::MultiPeek<I>, | ||
243 | } | ||
244 | |||
245 | // helper function | ||
246 | fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> { | ||
247 | if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt { | ||
248 | return Some(pp); | ||
249 | } | ||
250 | None | ||
251 | } | ||
252 | |||
253 | impl<'a, I> TokenPeek<'a, I> | ||
254 | where | ||
255 | I: Iterator<Item = &'a tt::TokenTree>, | ||
256 | { | ||
257 | pub fn new(iter: I) -> Self { | ||
258 | TokenPeek { iter: itertools::multipeek(iter) } | ||
259 | } | ||
260 | |||
261 | pub fn current_punct2(&mut self, p: tt::Punct) -> Option<((char, char), bool)> { | ||
262 | if p.spacing != tt::Spacing::Joint { | ||
263 | return None; | ||
264 | } | ||
265 | |||
266 | self.iter.reset_peek(); | ||
267 | let p1 = to_punct(self.iter.peek()?)?; | ||
268 | Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint)) | ||
269 | } | ||
270 | |||
271 | pub fn current_punct3(&mut self, p: tt::Punct) -> Option<((char, char, char), bool)> { | ||
272 | self.current_punct2(p).and_then(|((p0, p1), last_joint)| { | ||
273 | if !last_joint { | ||
274 | None | ||
275 | } else { | ||
276 | let p2 = to_punct(*self.iter.peek()?)?; | ||
277 | Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint)) | ||
278 | } | ||
279 | }) | ||
280 | } | ||
281 | } | ||
diff --git a/crates/ra_mbe/src/tt_iter.rs b/crates/ra_mbe/src/tt_iter.rs new file mode 100644 index 000000000..c53f99d1e --- /dev/null +++ b/crates/ra_mbe/src/tt_iter.rs | |||
@@ -0,0 +1,67 @@ | |||
1 | #[derive(Debug, Clone)] | ||
2 | pub(crate) struct TtIter<'a> { | ||
3 | pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>, | ||
4 | } | ||
5 | |||
6 | impl<'a> TtIter<'a> { | ||
7 | pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> { | ||
8 | TtIter { inner: subtree.token_trees.iter() } | ||
9 | } | ||
10 | |||
11 | pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> { | ||
12 | match self.next() { | ||
13 | Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => { | ||
14 | Ok(()) | ||
15 | } | ||
16 | _ => Err(()), | ||
17 | } | ||
18 | } | ||
19 | |||
20 | pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> { | ||
21 | match self.next() { | ||
22 | Some(tt::TokenTree::Subtree(it)) => Ok(it), | ||
23 | _ => Err(()), | ||
24 | } | ||
25 | } | ||
26 | |||
27 | pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> { | ||
28 | match self.next() { | ||
29 | Some(tt::TokenTree::Leaf(it)) => Ok(it), | ||
30 | _ => Err(()), | ||
31 | } | ||
32 | } | ||
33 | |||
34 | pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { | ||
35 | match self.expect_leaf()? { | ||
36 | tt::Leaf::Ident(it) => Ok(it), | ||
37 | _ => Err(()), | ||
38 | } | ||
39 | } | ||
40 | |||
41 | pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Literal, ()> { | ||
42 | match self.expect_leaf()? { | ||
43 | tt::Leaf::Literal(it) => Ok(it), | ||
44 | _ => Err(()), | ||
45 | } | ||
46 | } | ||
47 | |||
48 | pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> { | ||
49 | match self.expect_leaf()? { | ||
50 | tt::Leaf::Punct(it) => Ok(it), | ||
51 | _ => Err(()), | ||
52 | } | ||
53 | } | ||
54 | } | ||
55 | |||
56 | impl<'a> Iterator for TtIter<'a> { | ||
57 | type Item = &'a tt::TokenTree; | ||
58 | fn next(&mut self) -> Option<Self::Item> { | ||
59 | self.inner.next() | ||
60 | } | ||
61 | |||
62 | fn size_hint(&self) -> (usize, Option<usize>) { | ||
63 | self.inner.size_hint() | ||
64 | } | ||
65 | } | ||
66 | |||
67 | impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {} | ||