Merge #4029

4029: Fix various proc-macro bugs r=matklad a=edwin0cheng

This PRs does the following things:

1. Fixed #4001 by splitting `LIFETIME` lexer token to two mbe tokens. It is because rustc token stream expects `LIFETIME` as a combination of punct and ident, but RA `tt:TokenTree` treats it as a single `Ident` previously.
2. Fixed #4003, by skipping `proc-macro` for completion. It is because currently we don't have `AstNode` for `proc-macro`. We would need to redesign how to implement `HasSource` for `proc-macro`.
3.  Fixed a bug how empty `TokenStream` merging in `proc-macro-srv` such that no L_DOLLAR and R_DOLLAR will be emitted accidentally. 


Co-authored-by: Edwin Cheng <[email protected]>

1 files changed, 26 insertions, 8 deletions

diff --git a/crates/ra_mbe/src/mbe_expander/matcher.rs b/crates/ra_mbe/src/mbe_expander/matcher.rs
index 2579382da..9485c62b8 100644
--- a/crates/ra_mbe/src/mbe_expander/matcher.rs
+++ b/crates/ra_mbe/src/mbe_expander/matcher.rs
@@ -202,6 +202,13 @@ impl<'a> TtIter<'a> {
     }
 
     pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+        match self.peek_n(0) {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
+                return self.expect_lifetime();
+            }
+            _ => (),
+        }
+
         let tt = self.next().ok_or_else(|| ())?.clone();
         let punct = match tt {
             tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
@@ -255,13 +262,21 @@ impl<'a> TtIter<'a> {
         }
     }
 
-    pub(crate) fn expect_lifetime(&mut self) -> Result<&tt::Ident, ()> {
-        let ident = self.expect_ident()?;
-        // check if it start from "`"
-        if !ident.text.starts_with('\'') {
+    pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+        let punct = self.expect_punct()?;
+        if punct.char != '\'' {
             return Err(());
         }
-        Ok(ident)
+        let ident = self.expect_ident()?;
+
+        Ok(tt::Subtree {
+            delimiter: None,
+            token_trees: vec![
+                tt::Leaf::Punct(punct.clone()).into(),
+                tt::Leaf::Ident(ident.clone()).into(),
+            ],
+        }
+        .into())
     }
 
     pub(crate) fn expect_fragment(
@@ -274,7 +289,10 @@ impl<'a> TtIter<'a> {
         }
 
         impl<'a> TreeSink for OffsetTokenSink<'a> {
-            fn token(&mut self, _kind: SyntaxKind, n_tokens: u8) {
+            fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+                if kind == SyntaxKind::LIFETIME {
+                    n_tokens = 2;
+                }
                 for _ in 0..n_tokens {
                     self.cursor = self.cursor.bump_subtree();
                 }
@@ -286,7 +304,7 @@ impl<'a> TtIter<'a> {
             }
         }
 
-        let buffer = TokenBuffer::new(self.inner.as_slice());
+        let buffer = TokenBuffer::new(&self.inner.as_slice());
         let mut src = SubtreeTokenSource::new(&buffer);
         let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
 
@@ -422,7 +440,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragmen
                 "tt" => input.expect_tt().map(Some).map_err(|()| err!()),
                 "lifetime" => input
                     .expect_lifetime()
-                    .map(|ident| Some(tt::Leaf::Ident(ident.clone()).into()))
+                    .map(|tt| Some(tt))
                     .map_err(|()| err!("expected lifetime")),
                 "literal" => input
                     .expect_literal()