Merge #6158

6158: Fix for negative literals in macros r=matklad a=cutsoy

_This pull request fixes #6028._

When writing `-42.0f32` in Rust, it is usually parsed as two different tokens (a minus operator and a float literal).

But a procedural macro can also generate new tokens, including negative [float literals](https://doc.rust-lang.org/stable/proc_macro/struct.Literal.html#method.f32_suffixed):

```rust
#[proc_macro]
fn example_verbose(input: TokenStream) -> TokenStream {
    let literal = Literal::f32_suffixed(-42.0);
    quote! { #literal }
}
```

or even shorter

```rust
#[proc_macro]
fn example(input: TokenStream) -> TokenStream {
    let literal = -42.0f32;
    quote! { #literal }
}
```

Unfortunately, these currently cause RA to crash:

```
thread '<unnamed>' panicked at 'Fail to convert given literal Literal {
    text: "-42.0f32",
    id: TokenId(
        4294967295,
    ),
}', crates/mbe/src/subtree_source.rs:161:28
```

This pull request contains both a fix 8cf9362 and a unit test 27798ee. In addition, I installed the patched server with `cargo xtask install --server` and verified in VSCode that it no longer crashes when a procedural macro returns a negative number literal.

Co-authored-by: Tim <[email protected]>

1 files changed, 33 insertions, 5 deletions

diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs
index 41461b315..226dc3bec 100644
--- a/crates/mbe/src/subtree_source.rs
+++ b/crates/mbe/src/subtree_source.rs
@@ -2,7 +2,7 @@
 
 use parser::{Token, TokenSource};
 use std::cell::{Cell, Ref, RefCell};
-use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use syntax::{tokenize, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use tt::buffer::{Cursor, TokenBuffer};
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -155,10 +155,17 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
 }
 
 fn convert_literal(l: &tt::Literal) -> TtToken {
-    let kind = lex_single_syntax_kind(&l.text)
-        .map(|(kind, _error)| kind)
-        .filter(|kind| kind.is_literal())
-        .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
+    let mut kinds = tokenize(&l.text).0.into_iter().map(|token| token.kind);
+
+    let kind = match kinds.next() {
+        Some(kind) if kind.is_literal() => Some(kind),
+        Some(SyntaxKind::MINUS) => match kinds.next() {
+            Some(kind) if kind.is_literal() => Some(kind),
+            _ => None,
+        },
+        _ => None,
+    }
+    .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
 
     TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
 }
@@ -195,3 +202,24 @@ fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
         tt::Leaf::Punct(punct) => convert_punct(*punct),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{convert_literal, TtToken};
+    use syntax::{SmolStr, SyntaxKind};
+
+    #[test]
+    fn test_negative_literal() {
+        assert_eq!(
+            convert_literal(&tt::Literal {
+                id: tt::TokenId::unspecified(),
+                text: SmolStr::new("-42.0")
+            }),
+            TtToken {
+                kind: SyntaxKind::FLOAT_NUMBER,
+                is_joint_to_next: false,
+                text: SmolStr::new("-42.0")
+            }
+        );
+    }
+}