Merge #8122

8122: Make bare underscore token an Ident rather than Punct in proc-macro r=edwin0cheng a=kevinmehall

In rustc and proc-macro2, a bare `_` token is parsed for procedural macro purposes as `Ident` rather than `Punct` (see https://github.com/rust-lang/rust/pull/48842). This changes rust-analyzer to match rustc's behavior and implementation by handling `_` as an Ident in token trees, but explicitly preventing `$x:ident` from matching it in MBE.

proc macro crate:
```rust
#[proc_macro]
pub fn input(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
    dbg!(input)
}
```

test crate:
```rust
test_proc_macro::input!(_);
```

output (rustc):
```rust
[test-proc-macro/src/lib.rs:10] input = TokenStream [
    Ident {
        ident: "_",
        span: #0 bytes(173..174),
    },
]
```

output (rust-analyzer before this change):
```rust
[test-proc-macro/src/lib.rs:10] input = TokenStream [
    Punct {
        ch: '_',
        spacing: Joint,
        span: 4294967295,
    },
]
```

output (rust-analyzer after this change):
```rust
[test-proc-macro/src/lib.rs:10] input = TokenStream [
    Ident {
        ident: "_",
        span: 4294967295,
    },
]
```


Co-authored-by: Kevin Mehall <[email protected]>

1 files changed, 5 insertions, 2 deletions

diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
index 85163c4b3..8bba3d3d5 100644
--- a/crates/mbe/src/syntax_bridge.rs
+++ b/crates/mbe/src/syntax_bridge.rs
@@ -350,7 +350,7 @@ trait TokenConvertor {
             return;
         }
 
-        result.push(if k.is_punct() {
+        result.push(if k.is_punct() && k != UNDERSCORE {
             assert_eq!(range.len(), TextSize::of('.'));
             let delim = match k {
                 T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
@@ -395,7 +395,9 @@ trait TokenConvertor {
                     {
                         tt::Spacing::Alone
                     }
-                    Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
+                    Some(next) if next.kind().is_punct() && next.kind() != UNDERSCORE => {
+                        tt::Spacing::Joint
+                    }
                     _ => tt::Spacing::Alone,
                 };
                 let char = match token.to_char() {
@@ -415,6 +417,7 @@ trait TokenConvertor {
             let leaf: tt::Leaf = match k {
                 T![true] | T![false] => make_leaf!(Ident),
                 IDENT => make_leaf!(Ident),
+                UNDERSCORE => make_leaf!(Ident),
                 k if k.is_keyword() => make_leaf!(Ident),
                 k if k.is_literal() => make_leaf!(Literal),
                 LIFETIME_IDENT => {