aboutsummaryrefslogtreecommitdiff
path: root/crates/mbe/src
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2021-01-30 08:12:30 +0000
committerEdwin Cheng <[email protected]>2021-01-30 08:12:30 +0000
commit438b34dceede1cf0bebef9309d90ab6a3bde5002 (patch)
tree05bc6ba095810a565b1c734d2dc576b13f94f187 /crates/mbe/src
parente7108fb5b198d4fe416ce2408afaa86f1020c37d (diff)
Simpilfy mbe parsing
Diffstat (limited to 'crates/mbe/src')
-rw-r--r--crates/mbe/src/expander/matcher.rs94
-rw-r--r--crates/mbe/src/expander/transcriber.rs16
-rw-r--r--crates/mbe/src/lib.rs21
-rw-r--r--crates/mbe/src/parser.rs14
4 files changed, 71 insertions, 74 deletions
diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs
index 987a4f676..800931cd1 100644
--- a/crates/mbe/src/expander/matcher.rs
+++ b/crates/mbe/src/expander/matcher.rs
@@ -70,7 +70,7 @@ pub(super) struct Match {
70} 70}
71 71
72impl Match { 72impl Match {
73 pub(super) fn add_err(&mut self, err: ExpandError) { 73 fn add_err(&mut self, err: ExpandError) {
74 let prev_err = self.err.take(); 74 let prev_err = self.err.take();
75 self.err = prev_err.or(Some(err)); 75 self.err = prev_err.or(Some(err));
76 self.err_count += 1; 76 self.err_count += 1;
@@ -79,12 +79,10 @@ impl Match {
79 79
80/// Matching errors are added to the `Match`. 80/// Matching errors are added to the `Match`.
81pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Match { 81pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
82 assert!(pattern.delimiter == None);
83
84 let mut res = Match::default(); 82 let mut res = Match::default();
85 let mut src = TtIter::new(src); 83 let mut src = TtIter::new(src);
86 84
87 match_subtree(&mut res, pattern, &mut src); 85 match_tokens(&mut res, pattern, &mut src);
88 86
89 if src.len() > 0 { 87 if src.len() > 0 {
90 res.unmatched_tts += src.len(); 88 res.unmatched_tts += src.len();
@@ -94,36 +92,16 @@ pub(super) fn match_(pattern: &MetaTemplate, src: &tt::Subtree) -> Match {
94 res 92 res
95} 93}
96 94
97fn match_subtree(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) { 95fn match_tokens(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
98 for op in pattern.iter() { 96 for op in pattern.iter() {
99 match op { 97 match op {
100 Op::Leaf(lhs) => { 98 Op::Leaf(lhs) => {
101 let rhs = match src.expect_leaf() { 99 if let Err(err) = match_leaf(lhs, src) {
102 Ok(l) => l, 100 res.add_err(err);
103 Err(()) => { 101 continue;
104 res.add_err(err!("expected leaf: `{}`", lhs));
105 continue;
106 }
107 };
108 match (lhs, rhs) {
109 (
110 tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
111 tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
112 ) if lhs == rhs => (),
113 (
114 tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
115 tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
116 ) if lhs == rhs => (),
117 (
118 tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
119 tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
120 ) if lhs == rhs => (),
121 _ => {
122 res.add_err(ExpandError::UnexpectedToken);
123 }
124 } 102 }
125 } 103 }
126 Op::Subtree(lhs) => { 104 Op::Subtree { tokens, delimiter: delim } => {
127 let rhs = match src.expect_subtree() { 105 let rhs = match src.expect_subtree() {
128 Ok(s) => s, 106 Ok(s) => s,
129 Err(()) => { 107 Err(()) => {
@@ -131,12 +109,12 @@ fn match_subtree(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
131 continue; 109 continue;
132 } 110 }
133 }; 111 };
134 if lhs.delimiter_kind() != rhs.delimiter_kind() { 112 if delim.map(|it| it.kind) != rhs.delimiter_kind() {
135 res.add_err(err!("mismatched delimiter")); 113 res.add_err(err!("mismatched delimiter"));
136 continue; 114 continue;
137 } 115 }
138 let mut src = TtIter::new(rhs); 116 let mut src = TtIter::new(rhs);
139 match_subtree(res, lhs, &mut src); 117 match_tokens(res, tokens, &mut src);
140 if src.len() > 0 { 118 if src.len() > 0 {
141 res.add_err(err!("leftover tokens")); 119 res.add_err(err!("leftover tokens"));
142 } 120 }
@@ -162,14 +140,42 @@ fn match_subtree(res: &mut Match, pattern: &MetaTemplate, src: &mut TtIter) {
162 res.add_err(err); 140 res.add_err(err);
163 } 141 }
164 } 142 }
165 Op::Repeat { subtree, kind, separator } => { 143 Op::Repeat { tokens: subtree, kind, separator } => {
166 match_repeat(res, subtree, *kind, separator, src); 144 match_repeat(res, subtree, *kind, separator, src);
167 } 145 }
168 } 146 }
169 } 147 }
170} 148}
171 149
172pub(super) fn match_repeat( 150fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter) -> Result<(), ExpandError> {
151 let rhs = match src.expect_leaf() {
152 Ok(l) => l,
153 Err(()) => {
154 return Err(err!("expected leaf: `{}`", lhs));
155 }
156 };
157 match (lhs, rhs) {
158 (
159 tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
160 tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
161 ) if lhs == rhs => (),
162 (
163 tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
164 tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
165 ) if lhs == rhs => (),
166 (
167 tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
168 tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
169 ) if lhs == rhs => (),
170 _ => {
171 return Err(ExpandError::UnexpectedToken);
172 }
173 }
174
175 Ok(())
176}
177
178fn match_repeat(
173 res: &mut Match, 179 res: &mut Match,
174 pattern: &MetaTemplate, 180 pattern: &MetaTemplate,
175 kind: RepeatKind, 181 kind: RepeatKind,
@@ -191,7 +197,7 @@ pub(super) fn match_repeat(
191 } 197 }
192 198
193 let mut nested = Match::default(); 199 let mut nested = Match::default();
194 match_subtree(&mut nested, pattern, &mut fork); 200 match_tokens(&mut nested, pattern, &mut fork);
195 if nested.err.is_none() { 201 if nested.err.is_none() {
196 limit -= 1; 202 limit -= 1;
197 if limit == 0 { 203 if limit == 0 {
@@ -292,8 +298,8 @@ fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &MetaTemplate) {
292 match op { 298 match op {
293 Op::Var { name, .. } => buf.push(name.clone()), 299 Op::Var { name, .. } => buf.push(name.clone()),
294 Op::Leaf(_) => (), 300 Op::Leaf(_) => (),
295 Op::Subtree(subtree) => collect_vars(buf, subtree), 301 Op::Subtree { tokens, .. } => collect_vars(buf, tokens),
296 Op::Repeat { subtree, .. } => collect_vars(buf, subtree), 302 Op::Repeat { tokens, .. } => collect_vars(buf, tokens),
297 } 303 }
298 } 304 }
299} 305}
@@ -325,7 +331,7 @@ impl<'a> TtIter<'a> {
325 ok 331 ok
326 } 332 }
327 333
328 pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> { 334 fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
329 match self.peek_n(0) { 335 match self.peek_n(0) {
330 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { 336 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
331 return self.expect_lifetime(); 337 return self.expect_lifetime();
@@ -386,7 +392,7 @@ impl<'a> TtIter<'a> {
386 } 392 }
387 } 393 }
388 394
389 pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> { 395 fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
390 let punct = self.expect_punct()?; 396 let punct = self.expect_punct()?;
391 if punct.char != '\'' { 397 if punct.char != '\'' {
392 return Err(()); 398 return Err(());
@@ -403,13 +409,13 @@ impl<'a> TtIter<'a> {
403 .into()) 409 .into())
404 } 410 }
405 411
406 pub(crate) fn expect_fragment( 412 fn expect_fragment(
407 &mut self, 413 &mut self,
408 fragment_kind: parser::FragmentKind, 414 fragment_kind: parser::FragmentKind,
409 ) -> ExpandResult<Option<tt::TokenTree>> { 415 ) -> ExpandResult<Option<tt::TokenTree>> {
410 pub(crate) struct OffsetTokenSink<'a> { 416 struct OffsetTokenSink<'a> {
411 pub(crate) cursor: Cursor<'a>, 417 cursor: Cursor<'a>,
412 pub(crate) error: bool, 418 error: bool,
413 } 419 }
414 420
415 impl<'a> TreeSink for OffsetTokenSink<'a> { 421 impl<'a> TreeSink for OffsetTokenSink<'a> {
@@ -465,7 +471,7 @@ impl<'a> TtIter<'a> {
465 ExpandResult { value: res, err } 471 ExpandResult { value: res, err }
466 } 472 }
467 473
468 pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> { 474 fn eat_vis(&mut self) -> Option<tt::TokenTree> {
469 let mut fork = self.clone(); 475 let mut fork = self.clone();
470 match fork.expect_fragment(Visibility) { 476 match fork.expect_fragment(Visibility) {
471 ExpandResult { value: tt, err: None } => { 477 ExpandResult { value: tt, err: None } => {
@@ -476,7 +482,7 @@ impl<'a> TtIter<'a> {
476 } 482 }
477 } 483 }
478 484
479 pub(crate) fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> { 485 fn eat_char(&mut self, c: char) -> Option<tt::TokenTree> {
480 let mut fork = self.clone(); 486 let mut fork = self.clone();
481 match fork.expect_char(c) { 487 match fork.expect_char(c) {
482 Ok(_) => { 488 Ok(_) => {
diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs
index 30c090f32..78368a33e 100644
--- a/crates/mbe/src/expander/transcriber.rs
+++ b/crates/mbe/src/expander/transcriber.rs
@@ -2,6 +2,7 @@
2//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` 2//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
3 3
4use syntax::SmolStr; 4use syntax::SmolStr;
5use tt::Delimiter;
5 6
6use super::ExpandResult; 7use super::ExpandResult;
7use crate::{ 8use crate::{
@@ -54,10 +55,9 @@ pub(super) fn transcribe(
54 template: &MetaTemplate, 55 template: &MetaTemplate,
55 bindings: &Bindings, 56 bindings: &Bindings,
56) -> ExpandResult<tt::Subtree> { 57) -> ExpandResult<tt::Subtree> {
57 assert!(template.delimiter == None);
58 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() }; 58 let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() };
59 let mut arena: Vec<tt::TokenTree> = Vec::new(); 59 let mut arena: Vec<tt::TokenTree> = Vec::new();
60 expand_subtree(&mut ctx, template, &mut arena) 60 expand_subtree(&mut ctx, template, None, &mut arena)
61} 61}
62 62
63#[derive(Debug)] 63#[derive(Debug)]
@@ -80,6 +80,7 @@ struct ExpandCtx<'a> {
80fn expand_subtree( 80fn expand_subtree(
81 ctx: &mut ExpandCtx, 81 ctx: &mut ExpandCtx,
82 template: &MetaTemplate, 82 template: &MetaTemplate,
83 delimiter: Option<Delimiter>,
83 arena: &mut Vec<tt::TokenTree>, 84 arena: &mut Vec<tt::TokenTree>,
84) -> ExpandResult<tt::Subtree> { 85) -> ExpandResult<tt::Subtree> {
85 // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation 86 // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
@@ -88,8 +89,9 @@ fn expand_subtree(
88 for op in template.iter() { 89 for op in template.iter() {
89 match op { 90 match op {
90 Op::Leaf(tt) => arena.push(tt.clone().into()), 91 Op::Leaf(tt) => arena.push(tt.clone().into()),
91 Op::Subtree(tt) => { 92 Op::Subtree { tokens, delimiter } => {
92 let ExpandResult { value: tt, err: e } = expand_subtree(ctx, &tt, arena); 93 let ExpandResult { value: tt, err: e } =
94 expand_subtree(ctx, &tokens, *delimiter, arena);
93 err = err.or(e); 95 err = err.or(e);
94 arena.push(tt.into()); 96 arena.push(tt.into());
95 } 97 }
@@ -98,7 +100,7 @@ fn expand_subtree(
98 err = err.or(e); 100 err = err.or(e);
99 push_fragment(arena, fragment); 101 push_fragment(arena, fragment);
100 } 102 }
101 Op::Repeat { subtree, kind, separator } => { 103 Op::Repeat { tokens: subtree, kind, separator } => {
102 let ExpandResult { value: fragment, err: e } = 104 let ExpandResult { value: fragment, err: e } =
103 expand_repeat(ctx, subtree, *kind, separator, arena); 105 expand_repeat(ctx, subtree, *kind, separator, arena);
104 err = err.or(e); 106 err = err.or(e);
@@ -108,7 +110,7 @@ fn expand_subtree(
108 } 110 }
109 // drain the elements added in this instance of expand_subtree 111 // drain the elements added in this instance of expand_subtree
110 let tts = arena.drain(start_elements..arena.len()).collect(); 112 let tts = arena.drain(start_elements..arena.len()).collect();
111 ExpandResult { value: tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err } 113 ExpandResult { value: tt::Subtree { delimiter, token_trees: tts }, err }
112} 114}
113 115
114fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> { 116fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> {
@@ -162,7 +164,7 @@ fn expand_repeat(
162 let mut counter = 0; 164 let mut counter = 0;
163 165
164 loop { 166 loop {
165 let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, arena); 167 let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, None, arena);
166 let nesting_state = ctx.nesting.last_mut().unwrap(); 168 let nesting_state = ctx.nesting.last_mut().unwrap();
167 if nesting_state.at_end || !nesting_state.hit { 169 if nesting_state.at_end || !nesting_state.hit {
168 break; 170 break;
diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
index 56c632665..d80bd7a33 100644
--- a/crates/mbe/src/lib.rs
+++ b/crates/mbe/src/lib.rs
@@ -92,18 +92,11 @@ struct Rule {
92} 92}
93 93
94#[derive(Clone, Debug, PartialEq, Eq)] 94#[derive(Clone, Debug, PartialEq, Eq)]
95struct MetaTemplate { 95struct MetaTemplate(Vec<Op>);
96 delimiter: Option<Delimiter>,
97 tokens: Vec<Op>,
98}
99 96
100impl<'a> MetaTemplate { 97impl<'a> MetaTemplate {
101 fn iter(&self) -> impl Iterator<Item = &Op> { 98 fn iter(&self) -> impl Iterator<Item = &Op> {
102 self.tokens.iter() 99 self.0.iter()
103 }
104
105 fn delimiter_kind(&self) -> Option<DelimiterKind> {
106 self.delimiter.map(|it| it.kind)
107 } 100 }
108} 101}
109 102
@@ -288,8 +281,8 @@ impl Rule {
288 .expect_subtree() 281 .expect_subtree()
289 .map_err(|()| ParseError::Expected("expected subtree".to_string()))?; 282 .map_err(|()| ParseError::Expected("expected subtree".to_string()))?;
290 283
291 let lhs = MetaTemplate { tokens: parse_pattern(&lhs)?, delimiter: None }; 284 let lhs = MetaTemplate(parse_pattern(&lhs)?);
292 let rhs = MetaTemplate { tokens: parse_template(&rhs)?, delimiter: None }; 285 let rhs = MetaTemplate(parse_template(&rhs)?);
293 286
294 Ok(crate::Rule { lhs, rhs }) 287 Ok(crate::Rule { lhs, rhs })
295 } 288 }
@@ -298,8 +291,8 @@ impl Rule {
298fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> { 291fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
299 for op in pattern.iter() { 292 for op in pattern.iter() {
300 match op { 293 match op {
301 Op::Subtree(subtree) => validate(&subtree)?, 294 Op::Subtree { tokens, .. } => validate(&tokens)?,
302 Op::Repeat { subtree, separator, .. } => { 295 Op::Repeat { tokens: subtree, separator, .. } => {
303 // Checks that no repetition which could match an empty token 296 // Checks that no repetition which could match an empty token
304 // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558 297 // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558
305 298
@@ -319,7 +312,7 @@ fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
319 ) 312 )
320 } 313 }
321 Op::Leaf(_) => {} 314 Op::Leaf(_) => {}
322 Op::Subtree(_) => {} 315 Op::Subtree { .. } => {}
323 } 316 }
324 false 317 false
325 }) { 318 }) {
diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs
index b90ae7015..f891ec29c 100644
--- a/crates/mbe/src/parser.rs
+++ b/crates/mbe/src/parser.rs
@@ -3,15 +3,16 @@
3 3
4use smallvec::SmallVec; 4use smallvec::SmallVec;
5use syntax::SmolStr; 5use syntax::SmolStr;
6use tt::Delimiter;
6 7
7use crate::{tt_iter::TtIter, MetaTemplate, ParseError}; 8use crate::{tt_iter::TtIter, MetaTemplate, ParseError};
8 9
9#[derive(Clone, Debug, PartialEq, Eq)] 10#[derive(Clone, Debug, PartialEq, Eq)]
10pub(crate) enum Op { 11pub(crate) enum Op {
11 Var { name: SmolStr, kind: Option<SmolStr>, id: tt::TokenId }, 12 Var { name: SmolStr, kind: Option<SmolStr>, id: tt::TokenId },
12 Repeat { subtree: MetaTemplate, kind: RepeatKind, separator: Option<Separator> }, 13 Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
13 Leaf(tt::Leaf), 14 Leaf(tt::Leaf),
14 Subtree(MetaTemplate), 15 Subtree { tokens: MetaTemplate, delimiter: Option<Delimiter> },
15} 16}
16 17
17#[derive(Copy, Clone, Debug, PartialEq, Eq)] 18#[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -92,12 +93,10 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
92 match second { 93 match second {
93 tt::TokenTree::Subtree(subtree) => { 94 tt::TokenTree::Subtree(subtree) => {
94 let (separator, kind) = parse_repeat(src)?; 95 let (separator, kind) = parse_repeat(src)?;
95 let delimiter = subtree.delimiter;
96 let tokens = parse_inner(&subtree, mode) 96 let tokens = parse_inner(&subtree, mode)
97 .into_iter() 97 .into_iter()
98 .collect::<Result<Vec<Op>, ParseError>>()?; 98 .collect::<Result<Vec<Op>, ParseError>>()?;
99 let subtree = MetaTemplate { tokens, delimiter }; 99 Op::Repeat { tokens: MetaTemplate(tokens), separator, kind }
100 Op::Repeat { subtree, separator, kind }
101 } 100 }
102 tt::TokenTree::Leaf(leaf) => match leaf { 101 tt::TokenTree::Leaf(leaf) => match leaf {
103 tt::Leaf::Punct(punct) => { 102 tt::Leaf::Punct(punct) => {
@@ -136,12 +135,9 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
136 } 135 }
137 tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()), 136 tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()),
138 tt::TokenTree::Subtree(subtree) => { 137 tt::TokenTree::Subtree(subtree) => {
139 let delimiter = subtree.delimiter;
140 let tokens = 138 let tokens =
141 parse_inner(&subtree, mode).into_iter().collect::<Result<Vec<Op>, ParseError>>()?; 139 parse_inner(&subtree, mode).into_iter().collect::<Result<Vec<Op>, ParseError>>()?;
142 140 Op::Subtree { tokens: MetaTemplate(tokens), delimiter: subtree.delimiter }
143 let subtree = MetaTemplate { tokens, delimiter };
144 Op::Subtree(subtree)
145 } 141 }
146 }; 142 };
147 Ok(res) 143 Ok(res)