aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe
diff options
context:
space:
mode:
authorFlorian Diebold <[email protected]>2020-03-13 12:03:31 +0000
committerFlorian Diebold <[email protected]>2020-03-16 17:38:19 +0000
commitb973158aeb337041d4e1434cf5d8c609a0b02bef (patch)
treee3da1bb7a3c2d89623382865edc7ef64c039496d /crates/ra_mbe
parentd3773ec1522681de117d354f0c82e753c68c6d0b (diff)
Make MBE expansion more resilient (WIP)
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r--crates/ra_mbe/src/lib.rs4
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs29
-rw-r--r--crates/ra_mbe/src/mbe_expander/matcher.rs11
-rw-r--r--crates/ra_mbe/src/mbe_expander/transcriber.rs69
-rw-r--r--crates/ra_mbe/src/tests.rs3
5 files changed, 71 insertions, 45 deletions
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index 43afe24cc..3adec4978 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -30,6 +30,8 @@ pub enum ExpandError {
30 InvalidRepeat, 30 InvalidRepeat,
31} 31}
32 32
33pub type ExpandResult<T> = (T, Option<ExpandError>);
34
33pub use crate::syntax_bridge::{ 35pub use crate::syntax_bridge::{
34 ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node, 36 ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node,
35 TokenMap, 37 TokenMap,
@@ -150,7 +152,7 @@ impl MacroRules {
150 Ok(MacroRules { rules, shift: Shift::new(tt) }) 152 Ok(MacroRules { rules, shift: Shift::new(tt) })
151 } 153 }
152 154
153 pub fn expand(&self, tt: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 155 pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> {
154 // apply shift 156 // apply shift
155 let mut tt = tt.clone(); 157 let mut tt = tt.clone();
156 self.shift.shift_all(&mut tt); 158 self.shift.shift_all(&mut tt);
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index b455b7321..da3952428 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -8,19 +8,30 @@ mod transcriber;
8use ra_syntax::SmolStr; 8use ra_syntax::SmolStr;
9use rustc_hash::FxHashMap; 9use rustc_hash::FxHashMap;
10 10
11use crate::ExpandError; 11use crate::{ExpandResult, ExpandError};
12 12
13pub(crate) fn expand( 13pub(crate) fn expand(
14 rules: &crate::MacroRules, 14 rules: &crate::MacroRules,
15 input: &tt::Subtree, 15 input: &tt::Subtree,
16) -> Result<tt::Subtree, ExpandError> { 16) -> ExpandResult<tt::Subtree> {
17 rules.rules.iter().find_map(|it| expand_rule(it, input).ok()).ok_or(ExpandError::NoMatchingRule) 17 let (mut result, mut err) = (tt::Subtree::default(), Some(ExpandError::NoMatchingRule));
18 for rule in &rules.rules {
19 let (res, e) = expand_rule(rule, input);
20 if e.is_none() {
21 // if we find a rule that applies without errors, we're done
22 return (res, None);
23 }
24 // TODO decide which result is better
25 result = res;
26 err = e;
27 }
28 (result, err)
18} 29}
19 30
20fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 31fn expand_rule(rule: &crate::Rule, input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
21 let bindings = matcher::match_(&rule.lhs, input)?; 32 let (bindings, bindings_err) = dbg!(matcher::match_(&rule.lhs, input));
22 let res = transcriber::transcribe(&rule.rhs, &bindings)?; 33 let (res, transcribe_err) = dbg!(transcriber::transcribe(&rule.rhs, &bindings));
23 Ok(res) 34 (res, bindings_err.or(transcribe_err))
24} 35}
25 36
26/// The actual algorithm for expansion is not too hard, but is pretty tricky. 37/// The actual algorithm for expansion is not too hard, but is pretty tricky.
@@ -111,7 +122,7 @@ mod tests {
111 } 122 }
112 123
113 fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) { 124 fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) {
114 assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation), Err(err)); 125 assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation).1, Some(err));
115 } 126 }
116 127
117 fn format_macro(macro_body: &str) -> String { 128 fn format_macro(macro_body: &str) -> String {
@@ -138,7 +149,7 @@ mod tests {
138 fn expand_first( 149 fn expand_first(
139 rules: &crate::MacroRules, 150 rules: &crate::MacroRules,
140 invocation: &str, 151 invocation: &str,
141 ) -> Result<tt::Subtree, ExpandError> { 152 ) -> ExpandResult<tt::Subtree> {
142 let source_file = ast::SourceFile::parse(invocation).ok().unwrap(); 153 let source_file = ast::SourceFile::parse(invocation).ok().unwrap();
143 let macro_invocation = 154 let macro_invocation =
144 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); 155 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 49c53183a..f9d4952c6 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -11,6 +11,7 @@ use crate::{
11use ra_parser::{FragmentKind::*, TreeSink}; 11use ra_parser::{FragmentKind::*, TreeSink};
12use ra_syntax::{SmolStr, SyntaxKind}; 12use ra_syntax::{SmolStr, SyntaxKind};
13use tt::buffer::{Cursor, TokenBuffer}; 13use tt::buffer::{Cursor, TokenBuffer};
14use super::ExpandResult;
14 15
15impl Bindings { 16impl Bindings {
16 fn push_optional(&mut self, name: &SmolStr) { 17 fn push_optional(&mut self, name: &SmolStr) {
@@ -64,19 +65,19 @@ macro_rules! bail {
64 }; 65 };
65} 66}
66 67
67pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Bindings, ExpandError> { 68pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> ExpandResult<Bindings> {
68 assert!(pattern.delimiter == None); 69 assert!(pattern.delimiter == None);
69 70
70 let mut res = Bindings::default(); 71 let mut res = Bindings::default();
71 let mut src = TtIter::new(src); 72 let mut src = TtIter::new(src);
72 73
73 match_subtree(&mut res, pattern, &mut src)?; 74 let mut err = match_subtree(&mut res, pattern, &mut src).err();
74 75
75 if src.len() > 0 { 76 if src.len() > 0 && err.is_none() {
76 bail!("leftover tokens"); 77 err = Some(err!("leftover tokens"));
77 } 78 }
78 79
79 Ok(res) 80 (res, err)
80} 81}
81 82
82fn match_subtree( 83fn match_subtree(
diff --git a/crates/ra_mbe/src/mbe_expander/transcriber.rs b/crates/ra_mbe/src/mbe_expander/transcriber.rs
index 7662020f3..c53c2d35e 100644
--- a/crates/ra_mbe/src/mbe_expander/transcriber.rs
+++ b/crates/ra_mbe/src/mbe_expander/transcriber.rs
@@ -3,6 +3,7 @@
3 3
4use ra_syntax::SmolStr; 4use ra_syntax::SmolStr;
5 5
6use super::ExpandResult;
6use crate::{ 7use crate::{
7 mbe_expander::{Binding, Bindings, Fragment}, 8 mbe_expander::{Binding, Bindings, Fragment},
8 parser::{parse_template, Op, RepeatKind, Separator}, 9 parser::{parse_template, Op, RepeatKind, Separator},
@@ -49,10 +50,7 @@ impl Bindings {
49 } 50 }
50} 51}
51 52
52pub(super) fn transcribe( 53pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult<tt::Subtree> {
53 template: &tt::Subtree,
54 bindings: &Bindings,
55) -> Result<tt::Subtree, ExpandError> {
56 assert!(template.delimiter == None); 54 assert!(template.delimiter == None);
57 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; 55 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() };
58 expand_subtree(&mut ctx, template) 56 expand_subtree(&mut ctx, template)
@@ -75,35 +73,46 @@ struct ExpandCtx<'a> {
75 nesting: Vec<NestingState>, 73 nesting: Vec<NestingState>,
76} 74}
77 75
78fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> Result<tt::Subtree, ExpandError> { 76fn expand_subtree(ctx: &mut ExpandCtx, template: &tt::Subtree) -> ExpandResult<tt::Subtree> {
79 let mut buf: Vec<tt::TokenTree> = Vec::new(); 77 let mut buf: Vec<tt::TokenTree> = Vec::new();
78 let mut err = None;
80 for op in parse_template(template) { 79 for op in parse_template(template) {
81 match op? { 80 let op = match op {
81 Ok(op) => op,
82 Err(e) => {
83 err = Some(e);
84 break;
85 }
86 };
87 match op {
82 Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()), 88 Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => buf.push(tt.clone()),
83 Op::TokenTree(tt::TokenTree::Subtree(tt)) => { 89 Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
84 let tt = expand_subtree(ctx, tt)?; 90 let (tt, e) = expand_subtree(ctx, tt);
91 err = err.or(e);
85 buf.push(tt.into()); 92 buf.push(tt.into());
86 } 93 }
87 Op::Var { name, kind: _ } => { 94 Op::Var { name, kind: _ } => {
88 let fragment = expand_var(ctx, name)?; 95 let (fragment, e) = expand_var(ctx, name);
96 err = err.or(e);
89 push_fragment(&mut buf, fragment); 97 push_fragment(&mut buf, fragment);
90 } 98 }
91 Op::Repeat { subtree, kind, separator } => { 99 Op::Repeat { subtree, kind, separator } => {
92 let fragment = expand_repeat(ctx, subtree, kind, separator)?; 100 let (fragment, e) = expand_repeat(ctx, subtree, kind, separator);
101 err = err.or(e);
93 push_fragment(&mut buf, fragment) 102 push_fragment(&mut buf, fragment)
94 } 103 }
95 } 104 }
96 } 105 }
97 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf }) 106 (tt::Subtree { delimiter: template.delimiter, token_trees: buf }, err)
98} 107}
99 108
100fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError> { 109fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult<Fragment> {
101 let res = if v == "crate" { 110 if v == "crate" {
102 // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. 111 // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
103 let tt = 112 let tt =
104 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() }) 113 tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
105 .into(); 114 .into();
106 Fragment::Tokens(tt) 115 (Fragment::Tokens(tt), None)
107 } else if !ctx.bindings.contains(v) { 116 } else if !ctx.bindings.contains(v) {
108 // Note that it is possible to have a `$var` inside a macro which is not bound. 117 // Note that it is possible to have a `$var` inside a macro which is not bound.
109 // For example: 118 // For example:
@@ -132,11 +141,13 @@ fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> Result<Fragment, ExpandError>
132 ], 141 ],
133 } 142 }
134 .into(); 143 .into();
135 Fragment::Tokens(tt) 144 (Fragment::Tokens(tt), None)
136 } else { 145 } else {
137 ctx.bindings.get(&v, &mut ctx.nesting)?.clone() 146 ctx.bindings.get(&v, &mut ctx.nesting).map_or_else(
138 }; 147 |e| (Fragment::Tokens(tt::TokenTree::empty()), Some(e)),
139 Ok(res) 148 |b| (b.clone(), None),
149 )
150 }
140} 151}
141 152
142fn expand_repeat( 153fn expand_repeat(
@@ -144,17 +155,17 @@ fn expand_repeat(
144 template: &tt::Subtree, 155 template: &tt::Subtree,
145 kind: RepeatKind, 156 kind: RepeatKind,
146 separator: Option<Separator>, 157 separator: Option<Separator>,
147) -> Result<Fragment, ExpandError> { 158) -> ExpandResult<Fragment> {
148 let mut buf: Vec<tt::TokenTree> = Vec::new(); 159 let mut buf: Vec<tt::TokenTree> = Vec::new();
149 ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); 160 ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false });
150 // Dirty hack to make macro-expansion terminate. 161 // Dirty hack to make macro-expansion terminate.
151 // This should be replaced by a propper macro-by-example implementation 162 // This should be replaced by a proper macro-by-example implementation
152 let limit = 65536; 163 let limit = 65536;
153 let mut has_seps = 0; 164 let mut has_seps = 0;
154 let mut counter = 0; 165 let mut counter = 0;
155 166
156 loop { 167 loop {
157 let res = expand_subtree(ctx, template); 168 let (mut t, e) = expand_subtree(ctx, template);
158 let nesting_state = ctx.nesting.last_mut().unwrap(); 169 let nesting_state = ctx.nesting.last_mut().unwrap();
159 if nesting_state.at_end || !nesting_state.hit { 170 if nesting_state.at_end || !nesting_state.hit {
160 break; 171 break;
@@ -172,10 +183,10 @@ fn expand_repeat(
172 break; 183 break;
173 } 184 }
174 185
175 let mut t = match res { 186 if e.is_some() {
176 Ok(t) => t, 187 continue;
177 Err(_) => continue, 188 }
178 }; 189
179 t.delimiter = None; 190 t.delimiter = None;
180 push_subtree(&mut buf, t); 191 push_subtree(&mut buf, t);
181 192
@@ -209,14 +220,14 @@ fn expand_repeat(
209 buf.pop(); 220 buf.pop();
210 } 221 }
211 222
212 if RepeatKind::OneOrMore == kind && counter == 0 {
213 return Err(ExpandError::UnexpectedToken);
214 }
215
216 // Check if it is a single token subtree without any delimiter 223 // Check if it is a single token subtree without any delimiter
217 // e.g {Delimiter:None> ['>'] /Delimiter:None>} 224 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
218 let tt = tt::Subtree { delimiter: None, token_trees: buf }.into(); 225 let tt = tt::Subtree { delimiter: None, token_trees: buf }.into();
219 Ok(Fragment::Tokens(tt)) 226
227 if RepeatKind::OneOrMore == kind && counter == 0 {
228 return (Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken));
229 }
230 (Fragment::Tokens(tt), None)
220} 231}
221 232
222fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { 233fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 6d5d1e9e6..4d3140fa9 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -1430,7 +1430,8 @@ impl MacroFixture {
1430 let (invocation_tt, _) = 1430 let (invocation_tt, _) =
1431 ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap(); 1431 ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap();
1432 1432
1433 self.rules.expand(&invocation_tt) 1433 let (tt, err) = self.rules.expand(&invocation_tt);
1434 err.map(Err).unwrap_or(Ok(tt))
1434 } 1435 }
1435 1436
1436 fn assert_expand_err(&self, invocation: &str, err: &ExpandError) { 1437 fn assert_expand_err(&self, invocation: &str, err: &ExpandError) {