From ce4eeec1bf805bd0be95651d38f95340c5b8848a Mon Sep 17 00:00:00 2001 From: Edwin Cheng Date: Tue, 26 Jan 2021 05:15:47 +0800 Subject: Support Macro v2 in mbe --- crates/mbe/Cargo.toml | 3 +- crates/mbe/src/lib.rs | 72 ++++++++++++- crates/mbe/src/mbe_expander.rs | 9 +- crates/mbe/src/tests.rs | 233 +++++++++++++++++++++++++++-------------- 4 files changed, 228 insertions(+), 89 deletions(-) diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml index af80e2be3..43bc10490 100644 --- a/crates/mbe/Cargo.toml +++ b/crates/mbe/Cargo.toml @@ -17,6 +17,5 @@ log = "0.4.8" syntax = { path = "../syntax", version = "0.0.0" } parser = { path = "../parser", version = "0.0.0" } tt = { path = "../tt", version = "0.0.0" } - -[dev-dependencies] test_utils = { path = "../test_utils" } + diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index 19543d777..35cde5f10 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -14,6 +14,7 @@ mod tests; use std::fmt; +use test_utils::mark; pub use tt::{Delimiter, DelimiterKind, Punct}; use crate::{ @@ -76,6 +77,14 @@ pub struct MacroRules { shift: Shift, } +/// For Macro 2.0 +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroDef { + rules: Vec, + /// Highest id of the token we have in TokenMap + shift: Shift, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct Rule { lhs: MetaTemplate, @@ -179,7 +188,7 @@ impl MacroRules { let mut src = TtIter::new(tt); let mut rules = Vec::new(); while src.len() > 0 { - let rule = Rule::parse(&mut src)?; + let rule = Rule::parse(&mut src, true)?; rules.push(rule); if let Err(()) = src.expect_char(';') { if src.len() > 0 { @@ -200,7 +209,58 @@ impl MacroRules { // apply shift let mut tt = tt.clone(); self.shift.shift_all(&mut tt); - mbe_expander::expand(self, &tt) + mbe_expander::expand_rules(&self.rules, &tt) + } + + pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { + self.shift.shift(id) + } + + pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) { + match self.shift.unshift(id) { + Some(id) => (id, Origin::Call), + None => (id, Origin::Def), + } + } +} + +impl MacroDef { + pub fn parse(tt: &tt::Subtree) -> Result { + let mut src = TtIter::new(tt); + let mut rules = Vec::new(); + + if Some(tt::DelimiterKind::Brace) == tt.delimiter_kind() { + mark::hit!(parse_macro_def_rules); + while src.len() > 0 { + let rule = Rule::parse(&mut src, true)?; + rules.push(rule); + if let Err(()) = src.expect_char(';') { + if src.len() > 0 { + return Err(ParseError::Expected("expected `;`".to_string())); + } + break; + } + } + } else { + mark::hit!(parse_macro_def_simple); + let rule = Rule::parse(&mut src, false)?; + if src.len() != 0 { + return Err(ParseError::Expected("remain tokens in macro def".to_string())); + } + rules.push(rule); + } + for rule in rules.iter() { + validate(&rule.lhs)?; + } + + Ok(MacroDef { rules, shift: Shift::new(tt) }) + } + + pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult { + // apply shift + let mut tt = tt.clone(); + self.shift.shift_all(&mut tt); + mbe_expander::expand_rules(&self.rules, &tt) } pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId { @@ -216,12 +276,14 @@ impl MacroRules { } impl Rule { - fn parse(src: &mut TtIter) -> Result { + fn parse(src: &mut TtIter, expect_arrow: bool) -> Result { let lhs = src .expect_subtree() .map_err(|()| ParseError::Expected("expected subtree".to_string()))?; - src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?; - src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?; + if expect_arrow { + src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?; + src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?; + } let rhs = src .expect_subtree() .map_err(|()| ParseError::Expected("expected subtree".to_string()))?; diff --git a/crates/mbe/src/mbe_expander.rs b/crates/mbe/src/mbe_expander.rs index a80b73db4..802c8fb0f 100644 --- a/crates/mbe/src/mbe_expander.rs +++ b/crates/mbe/src/mbe_expander.rs @@ -10,11 +10,10 @@ use syntax::SmolStr; use crate::{ExpandError, ExpandResult}; -pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult { - expand_rules(&rules.rules, input) -} - -fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult { +pub(crate) fn expand_rules( + rules: &[crate::Rule], + input: &tt::Subtree, +) -> ExpandResult { let mut match_: Option<(matcher::Match, &crate::Rule)> = None; for rule in rules { let new_match = match matcher::match_(&rule.lhs, input) { diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs index bd2977ebd..8d978163d 100644 --- a/crates/mbe/src/tests.rs +++ b/crates/mbe/src/tests.rs @@ -6,7 +6,7 @@ use syntax::{ SyntaxKind::{ERROR, IDENT}, SyntaxNode, WalkEvent, T, }; -use test_utils::assert_eq_text; +use test_utils::{assert_eq_text, mark}; use super::*; @@ -675,6 +675,36 @@ fn test_match_literal() { .assert_expand_items("foo! ['('];", "fn foo () {}"); } +#[test] +fn test_parse_macro_def_simple() { + mark::check!(parse_macro_def_simple); + + parse_macro2( + r#" +macro foo($id:ident) { + fn $id() {} +} +"#, + ) + .assert_expand_items("foo!(bar);", "fn bar () {}"); +} + +#[test] +fn test_parse_macro_def_rules() { + mark::check!(parse_macro_def_rules); + + parse_macro2( + r#" +macro foo { + ($id:ident) => { + fn $id() {} + } +} +"#, + ) + .assert_expand_items("foo!(bar);", "fn bar () {}"); +} + // The following tests are port from intellij-rust directly // https://github.com/intellij-rust/intellij-rust/blob/c4e9feee4ad46e7953b1948c112533360b6087bb/src/test/kotlin/org/rust/lang/core/macros/RsMacroExpansionTest.kt @@ -1699,95 +1729,122 @@ pub(crate) struct MacroFixture { rules: MacroRules, } -impl MacroFixture { - pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree { - self.try_expand_tt(invocation).unwrap() - } - - fn try_expand_tt(&self, invocation: &str) -> Result { - let source_file = ast::SourceFile::parse(invocation).tree(); - let macro_invocation = - source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - - let (invocation_tt, _) = ast_to_token_tree(¯o_invocation.token_tree().unwrap()) - .ok_or_else(|| ExpandError::ConversionError)?; +pub(crate) struct MacroFixture2 { + rules: MacroDef, +} - self.rules.expand(&invocation_tt).result() - } +macro_rules! impl_fixture { + ($name:ident) => { + impl $name { + pub(crate) fn expand_tt(&self, invocation: &str) -> tt::Subtree { + self.try_expand_tt(invocation).unwrap() + } - fn assert_expand_err(&self, invocation: &str, err: &ExpandError) { - assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err)); - } + fn try_expand_tt(&self, invocation: &str) -> Result { + let source_file = ast::SourceFile::parse(invocation).tree(); + let macro_invocation = + source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); - fn expand_items(&self, invocation: &str) -> SyntaxNode { - let expanded = self.expand_tt(invocation); - token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node() - } + let (invocation_tt, _) = ast_to_token_tree(¯o_invocation.token_tree().unwrap()) + .ok_or_else(|| ExpandError::ConversionError)?; - fn expand_statements(&self, invocation: &str) -> SyntaxNode { - let expanded = self.expand_tt(invocation); - token_tree_to_syntax_node(&expanded, FragmentKind::Statements).unwrap().0.syntax_node() - } + self.rules.expand(&invocation_tt).result() + } - fn expand_expr(&self, invocation: &str) -> SyntaxNode { - let expanded = self.expand_tt(invocation); - token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node() - } + #[allow(unused)] + fn assert_expand_err(&self, invocation: &str, err: &ExpandError) { + assert_eq!(self.try_expand_tt(invocation).as_ref(), Err(err)); + } - fn assert_expand_tt(&self, invocation: &str, expected: &str) { - let expansion = self.expand_tt(invocation); - assert_eq!(expansion.to_string(), expected); - } + #[allow(unused)] + fn expand_items(&self, invocation: &str) -> SyntaxNode { + let expanded = self.expand_tt(invocation); + token_tree_to_syntax_node(&expanded, FragmentKind::Items).unwrap().0.syntax_node() + } - fn assert_expand(&self, invocation: &str, expected: &str) { - let expansion = self.expand_tt(invocation); - let actual = format!("{:?}", expansion); - test_utils::assert_eq_text!(&expected.trim(), &actual.trim()); - } + #[allow(unused)] + fn expand_statements(&self, invocation: &str) -> SyntaxNode { + let expanded = self.expand_tt(invocation); + token_tree_to_syntax_node(&expanded, FragmentKind::Statements) + .unwrap() + .0 + .syntax_node() + } - fn assert_expand_items(&self, invocation: &str, expected: &str) -> &MacroFixture { - self.assert_expansion(FragmentKind::Items, invocation, expected); - self - } + #[allow(unused)] + fn expand_expr(&self, invocation: &str) -> SyntaxNode { + let expanded = self.expand_tt(invocation); + token_tree_to_syntax_node(&expanded, FragmentKind::Expr).unwrap().0.syntax_node() + } - fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &MacroFixture { - self.assert_expansion(FragmentKind::Statements, invocation, expected); - self - } + #[allow(unused)] + fn assert_expand_tt(&self, invocation: &str, expected: &str) { + let expansion = self.expand_tt(invocation); + assert_eq!(expansion.to_string(), expected); + } - fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) { - let expanded = self.expand_tt(invocation); - assert_eq!(expanded.to_string(), expected); - - let expected = expected.replace("$crate", "C_C__C"); - - // wrap the given text to a macro call - let expected = { - let wrapped = format!("wrap_macro!( {} )", expected); - let wrapped = ast::SourceFile::parse(&wrapped); - let wrapped = - wrapped.tree().syntax().descendants().find_map(ast::TokenTree::cast).unwrap(); - let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0; - wrapped.delimiter = None; - wrapped - }; + #[allow(unused)] + fn assert_expand(&self, invocation: &str, expected: &str) { + let expansion = self.expand_tt(invocation); + let actual = format!("{:?}", expansion); + test_utils::assert_eq_text!(&expected.trim(), &actual.trim()); + } - let expanded_tree = token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node(); - let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string(); + fn assert_expand_items(&self, invocation: &str, expected: &str) -> &$name { + self.assert_expansion(FragmentKind::Items, invocation, expected); + self + } - let expected_tree = token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node(); - let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string(); + #[allow(unused)] + fn assert_expand_statements(&self, invocation: &str, expected: &str) -> &$name { + self.assert_expansion(FragmentKind::Statements, invocation, expected); + self + } - let expected_tree = expected_tree.replace("C_C__C", "$crate"); - assert_eq!( - expanded_tree, expected_tree, - "\nleft:\n{}\nright:\n{}", - expanded_tree, expected_tree, - ); - } + fn assert_expansion(&self, kind: FragmentKind, invocation: &str, expected: &str) { + let expanded = self.expand_tt(invocation); + assert_eq!(expanded.to_string(), expected); + + let expected = expected.replace("$crate", "C_C__C"); + + // wrap the given text to a macro call + let expected = { + let wrapped = format!("wrap_macro!( {} )", expected); + let wrapped = ast::SourceFile::parse(&wrapped); + let wrapped = wrapped + .tree() + .syntax() + .descendants() + .find_map(ast::TokenTree::cast) + .unwrap(); + let mut wrapped = ast_to_token_tree(&wrapped).unwrap().0; + wrapped.delimiter = None; + wrapped + }; + + let expanded_tree = + token_tree_to_syntax_node(&expanded, kind).unwrap().0.syntax_node(); + let expanded_tree = debug_dump_ignore_spaces(&expanded_tree).trim().to_string(); + + let expected_tree = + token_tree_to_syntax_node(&expected, kind).unwrap().0.syntax_node(); + let expected_tree = debug_dump_ignore_spaces(&expected_tree).trim().to_string(); + + let expected_tree = expected_tree.replace("C_C__C", "$crate"); + assert_eq!( + expanded_tree, expected_tree, + "\nleft:\n{}\nright:\n{}", + expanded_tree, expected_tree, + ); + } + } + }; } -fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree { +impl_fixture!(MacroFixture); +impl_fixture!(MacroFixture2); + +fn parse_macro_rules_to_tt(ra_fixture: &str) -> tt::Subtree { let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); let macro_definition = source_file.syntax().descendants().find_map(ast::MacroRules::cast).unwrap(); @@ -1804,14 +1861,36 @@ fn parse_macro_to_tt(ra_fixture: &str) -> tt::Subtree { definition_tt } +fn parse_macro_def_to_tt(ra_fixture: &str) -> tt::Subtree { + let source_file = ast::SourceFile::parse(ra_fixture).ok().unwrap(); + let macro_definition = + source_file.syntax().descendants().find_map(ast::MacroDef::cast).unwrap(); + + let (definition_tt, _) = ast_to_token_tree(¯o_definition.body().unwrap()).unwrap(); + + let parsed = + parse_to_token_tree(&ra_fixture[macro_definition.body().unwrap().syntax().text_range()]) + .unwrap() + .0; + assert_eq!(definition_tt, parsed); + + definition_tt +} + pub(crate) fn parse_macro(ra_fixture: &str) -> MacroFixture { - let definition_tt = parse_macro_to_tt(ra_fixture); + let definition_tt = parse_macro_rules_to_tt(ra_fixture); let rules = MacroRules::parse(&definition_tt).unwrap(); MacroFixture { rules } } +pub(crate) fn parse_macro2(ra_fixture: &str) -> MacroFixture2 { + let definition_tt = parse_macro_def_to_tt(ra_fixture); + let rules = MacroDef::parse(&definition_tt).unwrap(); + MacroFixture2 { rules } +} + pub(crate) fn parse_macro_error(ra_fixture: &str) -> ParseError { - let definition_tt = parse_macro_to_tt(ra_fixture); + let definition_tt = parse_macro_rules_to_tt(ra_fixture); match MacroRules::parse(&definition_tt) { Ok(_) => panic!("Expect error"), -- cgit v1.2.3