aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe
diff options
context:
space:
mode:
authorbors[bot] <26634292+bors[bot]@users.noreply.github.com>2019-09-10 14:01:44 +0100
committerGitHub <[email protected]>2019-09-10 14:01:44 +0100
commit9d3c78e2eee6635772c99d7351b621cefb08bac5 (patch)
tree14e74b81c7195bcd5b308e799c3fd447e76274ca /crates/ra_mbe
parente2ebb467bdf3ebb7d29260adb95c56594c6db282 (diff)
parented726081d1df2fb6a1c21101996dcae203f79021 (diff)
Merge #1801
1801: WIP: switch to fully decomposed tokens internally r=matklad a=matklad Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_mbe')
-rw-r--r--crates/ra_mbe/Cargo.toml4
-rw-r--r--crates/ra_mbe/src/mbe_expander.rs50
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs9
-rw-r--r--crates/ra_mbe/src/tests.rs9
4 files changed, 45 insertions, 27 deletions
diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml
index 68f559295..b058dde91 100644
--- a/crates/ra_mbe/Cargo.toml
+++ b/crates/ra_mbe/Cargo.toml
@@ -12,3 +12,7 @@ itertools = "0.8.0"
12rustc-hash = "1.0.0" 12rustc-hash = "1.0.0"
13smallvec = "0.6.9" 13smallvec = "0.6.9"
14log = "0.4.5" 14log = "0.4.5"
15
16[dev-dependencies]
17test_utils = { path = "../test_utils" }
18
diff --git a/crates/ra_mbe/src/mbe_expander.rs b/crates/ra_mbe/src/mbe_expander.rs
index 01641fdee..08b0519d2 100644
--- a/crates/ra_mbe/src/mbe_expander.rs
+++ b/crates/ra_mbe/src/mbe_expander.rs
@@ -339,21 +339,13 @@ fn expand_subtree(
339 template: &crate::Subtree, 339 template: &crate::Subtree,
340 ctx: &mut ExpandCtx, 340 ctx: &mut ExpandCtx,
341) -> Result<tt::Subtree, ExpandError> { 341) -> Result<tt::Subtree, ExpandError> {
342 let token_trees = template 342 let mut buf: Vec<tt::TokenTree> = Vec::new();
343 .token_trees 343 for tt in template.token_trees.iter() {
344 .iter() 344 let tt = expand_tt(tt, ctx)?;
345 .map(|it| expand_tt(it, ctx)) 345 push_tt(&mut buf, tt);
346 .filter(|it| { 346 }
347 // Filter empty subtree
348 if let Ok(tt::TokenTree::Subtree(subtree)) = it {
349 subtree.delimiter != tt::Delimiter::None || !subtree.token_trees.is_empty()
350 } else {
351 true
352 }
353 })
354 .collect::<Result<Vec<_>, ExpandError>>()?;
355 347
356 Ok(tt::Subtree { token_trees, delimiter: template.delimiter }) 348 Ok(tt::Subtree { delimiter: template.delimiter, token_trees: buf })
357} 349}
358 350
359/// Reduce single token subtree to single token 351/// Reduce single token subtree to single token
@@ -377,7 +369,7 @@ fn expand_tt(
377 let res: tt::TokenTree = match template { 369 let res: tt::TokenTree = match template {
378 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(), 370 crate::TokenTree::Subtree(subtree) => expand_subtree(subtree, ctx)?.into(),
379 crate::TokenTree::Repeat(repeat) => { 371 crate::TokenTree::Repeat(repeat) => {
380 let mut token_trees: Vec<tt::TokenTree> = Vec::new(); 372 let mut buf: Vec<tt::TokenTree> = Vec::new();
381 ctx.nesting.push(0); 373 ctx.nesting.push(0);
382 // Dirty hack to make macro-expansion terminate. 374 // Dirty hack to make macro-expansion terminate.
383 // This should be replaced by a propper macro-by-example implementation 375 // This should be replaced by a propper macro-by-example implementation
@@ -418,23 +410,23 @@ fn expand_tt(
418 410
419 let idx = ctx.nesting.pop().unwrap(); 411 let idx = ctx.nesting.pop().unwrap();
420 ctx.nesting.push(idx + 1); 412 ctx.nesting.push(idx + 1);
421 token_trees.push(reduce_single_token(t)); 413 push_subtree(&mut buf, t);
422 414
423 if let Some(ref sep) = repeat.separator { 415 if let Some(ref sep) = repeat.separator {
424 match sep { 416 match sep {
425 crate::Separator::Ident(ident) => { 417 crate::Separator::Ident(ident) => {
426 has_seps = 1; 418 has_seps = 1;
427 token_trees.push(tt::Leaf::from(ident.clone()).into()); 419 buf.push(tt::Leaf::from(ident.clone()).into());
428 } 420 }
429 crate::Separator::Literal(lit) => { 421 crate::Separator::Literal(lit) => {
430 has_seps = 1; 422 has_seps = 1;
431 token_trees.push(tt::Leaf::from(lit.clone()).into()); 423 buf.push(tt::Leaf::from(lit.clone()).into());
432 } 424 }
433 425
434 crate::Separator::Puncts(puncts) => { 426 crate::Separator::Puncts(puncts) => {
435 has_seps = puncts.len(); 427 has_seps = puncts.len();
436 for punct in puncts { 428 for punct in puncts {
437 token_trees.push(tt::Leaf::from(*punct).into()); 429 buf.push(tt::Leaf::from(*punct).into());
438 } 430 }
439 } 431 }
440 } 432 }
@@ -450,16 +442,16 @@ fn expand_tt(
450 442
451 ctx.nesting.pop().unwrap(); 443 ctx.nesting.pop().unwrap();
452 for _ in 0..has_seps { 444 for _ in 0..has_seps {
453 token_trees.pop(); 445 buf.pop();
454 } 446 }
455 447
456 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 { 448 if crate::RepeatKind::OneOrMore == repeat.kind && counter == 0 {
457 return Err(ExpandError::UnexpectedToken); 449 return Err(ExpandError::UnexpectedToken);
458 } 450 }
459 451
460 // Check if it is a singel token subtree without any delimiter 452 // Check if it is a single token subtree without any delimiter
461 // e.g {Delimiter:None> ['>'] /Delimiter:None>} 453 // e.g {Delimiter:None> ['>'] /Delimiter:None>}
462 reduce_single_token(tt::Subtree { token_trees, delimiter: tt::Delimiter::None }) 454 reduce_single_token(tt::Subtree { delimiter: tt::Delimiter::None, token_trees: buf })
463 } 455 }
464 crate::TokenTree::Leaf(leaf) => match leaf { 456 crate::TokenTree::Leaf(leaf) => match leaf {
465 crate::Leaf::Ident(ident) => { 457 crate::Leaf::Ident(ident) => {
@@ -586,3 +578,17 @@ mod tests {
586 expand_rule(&rules.rules[0], &invocation_tt) 578 expand_rule(&rules.rules[0], &invocation_tt)
587 } 579 }
588} 580}
581
582fn push_tt(buf: &mut Vec<tt::TokenTree>, tt: tt::TokenTree) {
583 match tt {
584 tt::TokenTree::Subtree(tt) => push_subtree(buf, tt),
585 _ => buf.push(tt),
586 }
587}
588
589fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
590 match tt.delimiter {
591 tt::Delimiter::None => buf.extend(tt.token_trees),
592 _ => buf.push(tt.into()),
593 }
594}
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index a380b1cfd..26524adf9 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -70,7 +70,14 @@ fn fragment_to_syntax_node(
70 tt: &tt::Subtree, 70 tt: &tt::Subtree,
71 fragment_kind: FragmentKind, 71 fragment_kind: FragmentKind,
72) -> Result<Parse<SyntaxNode>, ExpandError> { 72) -> Result<Parse<SyntaxNode>, ExpandError> {
73 let tokens = [tt.clone().into()]; 73 let tmp;
74 let tokens = match tt {
75 tt::Subtree { delimiter: tt::Delimiter::None, token_trees } => token_trees.as_slice(),
76 _ => {
77 tmp = [tt.clone().into()];
78 &tmp[..]
79 }
80 };
74 let buffer = TokenBuffer::new(&tokens); 81 let buffer = TokenBuffer::new(&tokens);
75 let mut token_source = SubtreeTokenSource::new(&buffer); 82 let mut token_source = SubtreeTokenSource::new(&buffer);
76 let mut tree_sink = TtTreeSink::new(buffer.begin()); 83 let mut tree_sink = TtTreeSink::new(buffer.begin());
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 034ea639b..2b80c5f49 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -1,4 +1,5 @@
1use ra_syntax::{ast, AstNode, NodeOrToken}; 1use ra_syntax::{ast, AstNode, NodeOrToken};
2use test_utils::assert_eq_text;
2 3
3use super::*; 4use super::*;
4 5
@@ -152,7 +153,6 @@ pub(crate) fn assert_expansion(
152 153
153 // wrap the given text to a macro call 154 // wrap the given text to a macro call
154 let expected = text_to_tokentree(&expected); 155 let expected = text_to_tokentree(&expected);
155
156 let (expanded_tree, expected_tree) = match kind { 156 let (expanded_tree, expected_tree) = match kind {
157 MacroKind::Items => { 157 MacroKind::Items => {
158 let expanded_tree = token_tree_to_macro_items(&expanded).unwrap().tree(); 158 let expanded_tree = token_tree_to_macro_items(&expanded).unwrap().tree();
@@ -178,7 +178,7 @@ pub(crate) fn assert_expansion(
178 let expected_tree = expected_tree.replace("C_C__C", "$crate"); 178 let expected_tree = expected_tree.replace("C_C__C", "$crate");
179 assert_eq!( 179 assert_eq!(
180 expanded_tree, expected_tree, 180 expanded_tree, expected_tree,
181 "left => {}\nright => {}", 181 "\nleft:\n{}\nright:\n{}",
182 expanded_tree, expected_tree, 182 expanded_tree, expected_tree,
183 ); 183 );
184 184
@@ -657,6 +657,7 @@ fn test_expr() {
657} 657}
658 658
659#[test] 659#[test]
660#[ignore]
660fn test_expr_order() { 661fn test_expr_order() {
661 let rules = create_rules( 662 let rules = create_rules(
662 r#" 663 r#"
@@ -668,8 +669,8 @@ fn test_expr_order() {
668"#, 669"#,
669 ); 670 );
670 671
671 assert_eq!( 672 assert_eq_text!(
672 format!("{:#?}", expand_to_items(&rules, "foo! { 1 + 1 }").syntax()).trim(), 673 &format!("{:#?}", expand_to_items(&rules, "foo! { 1 + 1 }").syntax()),
673 r#"MACRO_ITEMS@[0; 15) 674 r#"MACRO_ITEMS@[0; 15)
674 FN_DEF@[0; 15) 675 FN_DEF@[0; 15)
675 FN_KW@[0; 2) "fn" 676 FN_KW@[0; 2) "fn"