Merge #4029

4029: Fix various proc-macro bugs r=matklad a=edwin0cheng

This PRs does the following things:

1. Fixed #4001 by splitting `LIFETIME` lexer token to two mbe tokens. It is because rustc token stream expects `LIFETIME` as a combination of punct and ident, but RA `tt:TokenTree` treats it as a single `Ident` previously.
2. Fixed #4003, by skipping `proc-macro` for completion. It is because currently we don't have `AstNode` for `proc-macro`. We would need to redesign how to implement `HasSource` for `proc-macro`.
3.  Fixed a bug how empty `TokenStream` merging in `proc-macro-srv` such that no L_DOLLAR and R_DOLLAR will be emitted accidentally. 


Co-authored-by: Edwin Cheng <[email protected]>

1 files changed, 31 insertions, 1 deletions

diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 91e324db9..46791efaa 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -50,6 +50,26 @@ impl<'a> SubtreeTokenSource<'a> {
     }
 
     fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
+        fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
+            let tkn = c.token_tree();
+
+            if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn {
+                if punct.char == '\'' {
+                    let next = c.bump();
+                    if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() {
+                        let res_cursor = next.bump();
+                        let text = SmolStr::new("'".to_string() + &ident.to_string());
+
+                        return Some((res_cursor, text));
+                    } else {
+                        panic!("Next token must be ident : {:#?}", next.token_tree());
+                    }
+                }
+            }
+
+            None
+        }
+
         if pos < self.cached.borrow().len() {
             return Ref::map(self.cached.borrow(), |c| &c[pos]);
         }
@@ -63,6 +83,12 @@ impl<'a> SubtreeTokenSource<'a> {
                     continue;
                 }
 
+                if let Some((curr, text)) = is_lifetime(cursor) {
+                    cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text }));
+                    self.cached_cursor.set(curr);
+                    continue;
+                }
+
                 match cursor.token_tree() {
                     Some(tt::TokenTree::Leaf(leaf)) => {
                         cached.push(Some(convert_leaf(&leaf)));
@@ -152,7 +178,11 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
 }
 
 fn convert_punct(p: tt::Punct) -> TtToken {
-    let kind = SyntaxKind::from_char(p.char).unwrap();
+    let kind = match SyntaxKind::from_char(p.char) {
+        None => panic!("{:#?} is not a valid punct", p),
+        Some(kind) => kind,
+    };
+
     let text = {
         let mut buf = [0u8; 4];
         let s: &str = p.char.encode_utf8(&mut buf);