From 0a0e22235b7ad222be1aaa7765b580f4096c9aeb Mon Sep 17 00:00:00 2001 From: Kevin Mehall Date: Sat, 20 Mar 2021 12:18:57 -0600 Subject: Make bare underscore token an Ident rather than Punct in proc-macro --- crates/mbe/src/expander/matcher.rs | 1 + crates/mbe/src/parser.rs | 12 ++---------- crates/mbe/src/subtree_source.rs | 1 + crates/mbe/src/syntax_bridge.rs | 7 +++++-- crates/mbe/src/tests/expand.rs | 6 ++++++ crates/proc_macro_srv/src/rustc_server.rs | 9 +++++++++ 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index b6782b4ba..3c53960ce 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -710,6 +710,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult input .expect_ident() + .and_then(|ident| if ident.text == "_" { Err(()) } else { Ok(ident) }) .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) .map_err(|()| err!("expected ident")), "tt" => input.expect_tt().map(Some).map_err(|()| err!()), diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 7b5b8ec16..c88387653 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -177,16 +177,8 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul Op::Repeat { tokens: MetaTemplate(tokens), separator, kind } } tt::TokenTree::Leaf(leaf) => match leaf { - tt::Leaf::Punct(punct) => { - static UNDERSCORE: SmolStr = SmolStr::new_inline("_"); - - if punct.char != '_' { - return Err(ParseError::Expected("_".to_string())); - } - let name = UNDERSCORE.clone(); - let kind = eat_fragment_kind(src, mode)?; - let id = punct.id; - Op::Var { name, kind, id } + tt::Leaf::Punct(_) => { + return Err(ParseError::Expected("ident".to_string())); } tt::Leaf::Ident(ident) if ident.text == "crate" => { // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path. diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs index d7433bd35..a05cab0f3 100644 --- a/crates/mbe/src/subtree_source.rs +++ b/crates/mbe/src/subtree_source.rs @@ -150,6 +150,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken { let kind = match ident.text.as_ref() { "true" => T![true], "false" => T![false], + "_" => UNDERSCORE, i if i.starts_with('\'') => LIFETIME_IDENT, _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT), }; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 85163c4b3..8bba3d3d5 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -350,7 +350,7 @@ trait TokenConvertor { return; } - result.push(if k.is_punct() { + result.push(if k.is_punct() && k != UNDERSCORE { assert_eq!(range.len(), TextSize::of('.')); let delim = match k { T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])), @@ -395,7 +395,9 @@ trait TokenConvertor { { tt::Spacing::Alone } - Some(next) if next.kind().is_punct() => tt::Spacing::Joint, + Some(next) if next.kind().is_punct() && next.kind() != UNDERSCORE => { + tt::Spacing::Joint + } _ => tt::Spacing::Alone, }; let char = match token.to_char() { @@ -415,6 +417,7 @@ trait TokenConvertor { let leaf: tt::Leaf = match k { T![true] | T![false] => make_leaf!(Ident), IDENT => make_leaf!(Ident), + UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), k if k.is_literal() => make_leaf!(Literal), LIFETIME_IDENT => { diff --git a/crates/mbe/src/tests/expand.rs b/crates/mbe/src/tests/expand.rs index 9dd8ff75b..2cce62781 100644 --- a/crates/mbe/src/tests/expand.rs +++ b/crates/mbe/src/tests/expand.rs @@ -1079,6 +1079,12 @@ macro_rules! q { .assert_expand_items(r#"q![_]"#, r#"0"#); } +#[test] +fn test_underscore_lifetime() { + parse_macro(r#"macro_rules! q { ($a:lifetime) => {0}; }"#) + .assert_expand_items(r#"q!['_]"#, r#"0"#); +} + #[test] fn test_vertical_bar_with_pat() { parse_macro( diff --git a/crates/proc_macro_srv/src/rustc_server.rs b/crates/proc_macro_srv/src/rustc_server.rs index ceefd187d..c147484c0 100644 --- a/crates/proc_macro_srv/src/rustc_server.rs +++ b/crates/proc_macro_srv/src/rustc_server.rs @@ -805,5 +805,14 @@ mod tests { let t2 = TokenStream::from_str("(a);").unwrap(); assert_eq!(t2.token_trees.len(), 2); assert_eq!(t2.token_trees[0], subtree_paren_a); + + let underscore = TokenStream::from_str("_").unwrap(); + assert_eq!( + underscore.token_trees[0], + tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: "_".into(), + id: tt::TokenId::unspecified(), + })) + ); } } -- cgit v1.2.3 From 0a7f28620a7002f47890c2030862052bcbf25cdb Mon Sep 17 00:00:00 2001 From: Kevin Mehall Date: Sat, 20 Mar 2021 17:43:51 -0600 Subject: Fix and test edge cases of `_` as ident --- crates/mbe/src/expander/matcher.rs | 5 ++--- crates/mbe/src/tests/rule.rs | 4 ++++ crates/mbe/src/tt_iter.rs | 7 +++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 3c53960ce..1682b21b0 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -710,7 +710,6 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult input .expect_ident() - .and_then(|ident| if ident.text == "_" { Err(()) } else { Ok(ident) }) .map(|ident| Some(tt::Leaf::from(ident.clone()).into())) .map_err(|()| err!("expected ident")), "tt" => input.expect_tt().map(Some).map_err(|()| err!()), @@ -763,7 +762,7 @@ impl<'a> TtIter<'a> { fn expect_separator(&mut self, separator: &Separator, idx: usize) -> bool { let mut fork = self.clone(); let ok = match separator { - Separator::Ident(lhs) if idx == 0 => match fork.expect_ident() { + Separator::Ident(lhs) if idx == 0 => match fork.expect_ident_or_underscore() { Ok(rhs) => rhs.text == lhs.text, _ => false, }, @@ -853,7 +852,7 @@ impl<'a> TtIter<'a> { if punct.char != '\'' { return Err(()); } - let ident = self.expect_ident()?; + let ident = self.expect_ident_or_underscore()?; Ok(tt::Subtree { delimiter: None, diff --git a/crates/mbe/src/tests/rule.rs b/crates/mbe/src/tests/rule.rs index 07277966d..bf48112b3 100644 --- a/crates/mbe/src/tests/rule.rs +++ b/crates/mbe/src/tests/rule.rs @@ -12,6 +12,9 @@ fn test_valid_arms() { } check("($i:ident) => ()"); + check("($(x),*) => ()"); + check("($(x)_*) => ()"); + check("($(x)i*) => ()"); check("($($i:ident)*) => ($_)"); check("($($true:ident)*) => ($true)"); check("($($false:ident)*) => ($false)"); @@ -32,6 +35,7 @@ fn test_invalid_arms() { check("($i) => ($i)", ParseError::UnexpectedToken("bad fragment specifier 1".into())); check("($i:) => ($i)", ParseError::UnexpectedToken("bad fragment specifier 1".into())); + check("($i:_) => ()", ParseError::UnexpectedToken("bad fragment specifier 1".into())); } fn parse_macro_arm(arm_definition: &str) -> Result { diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index a362d31fc..319a40f2a 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -49,6 +49,13 @@ impl<'a> TtIter<'a> { } pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> { + match self.expect_leaf()? { + tt::Leaf::Ident(it) if it.text != "_" => Ok(it), + _ => Err(()), + } + } + + pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> { match self.expect_leaf()? { tt::Leaf::Ident(it) => Ok(it), _ => Err(()), -- cgit v1.2.3