Merge #4029

4029: Fix various proc-macro bugs r=matklad a=edwin0cheng

This PRs does the following things:

1. Fixed #4001 by splitting `LIFETIME` lexer token to two mbe tokens. It is because rustc token stream expects `LIFETIME` as a combination of punct and ident, but RA `tt:TokenTree` treats it as a single `Ident` previously.
2. Fixed #4003, by skipping `proc-macro` for completion. It is because currently we don't have `AstNode` for `proc-macro`. We would need to redesign how to implement `HasSource` for `proc-macro`.
3.  Fixed a bug how empty `TokenStream` merging in `proc-macro-srv` such that no L_DOLLAR and R_DOLLAR will be emitted accidentally. 


Co-authored-by: Edwin Cheng <[email protected]>

1 files changed, 32 insertions, 5 deletions

diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 31e9b22e7..70899bc5d 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -271,7 +271,7 @@ struct RawConvertor<'a> {
     inner: std::slice::Iter<'a, RawToken>,
 }
 
-trait SrcToken {
+trait SrcToken: std::fmt::Debug {
     fn kind(&self) -> SyntaxKind;
 
     fn to_char(&self) -> Option<char>;
@@ -361,8 +361,12 @@ trait TokenConvertor {
                     Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
                     _ => tt::Spacing::Alone,
                 };
-                let char = token.to_char().expect("Token from lexer must be single char");
-
+                let char = match token.to_char() {
+                    Some(c) => c,
+                    None => {
+                        panic!("Token from lexer must be single char: token = {:#?}", token);
+                    }
+                };
                 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
             }
         } else {
@@ -373,9 +377,28 @@ trait TokenConvertor {
             }
             let leaf: tt::Leaf = match k {
                 T![true] | T![false] => make_leaf!(Literal),
-                IDENT | LIFETIME => make_leaf!(Ident),
+                IDENT => make_leaf!(Ident),
                 k if k.is_keyword() => make_leaf!(Ident),
                 k if k.is_literal() => make_leaf!(Literal),
+                LIFETIME => {
+                    let char_unit = TextUnit::from_usize(1);
+                    let r = TextRange::offset_len(range.start(), char_unit);
+                    let apostrophe = tt::Leaf::from(tt::Punct {
+                        char: '\'',
+                        spacing: tt::Spacing::Joint,
+                        id: self.id_alloc().alloc(r),
+                    });
+                    result.push(apostrophe.into());
+
+                    let r =
+                        TextRange::offset_len(range.start() + char_unit, range.len() - char_unit);
+                    let ident = tt::Leaf::from(tt::Ident {
+                        text: SmolStr::new(&token.to_text()[1..]),
+                        id: self.id_alloc().alloc(r),
+                    });
+                    result.push(ident.into());
+                    return;
+                }
                 _ => return,
             };
 
@@ -455,6 +478,7 @@ impl Convertor {
     }
 }
 
+#[derive(Debug)]
 enum SynToken {
     Ordiniary(SyntaxToken),
     Punch(SyntaxToken, TextUnit),
@@ -592,11 +616,14 @@ fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> SmolStr {
 }
 
 impl<'a> TreeSink for TtTreeSink<'a> {
-    fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
+    fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
         if kind == L_DOLLAR || kind == R_DOLLAR {
             self.cursor = self.cursor.bump_subtree();
             return;
         }
+        if kind == LIFETIME {
+            n_tokens = 2;
+        }
 
         let mut last = self.cursor;
         for _ in 0..n_tokens {