Merge #1148

1148: Add token_tree_to_xxx functions r=matklad a=edwin0cheng

<del>As discus in  PR #1147 , this PR added a `mbe::MacroKind` .
Currently only 2 kind of macro are supported, `SourceFile` and `Block`.</del>

Added following functions for `tt::TokenTree` and `ast::Node` conversion:

* token_tree_to_expr
* token_tree_to_pat
* token_tree_to_ty
* token_tree_to_macro_stmts
* token_tree_to_macro_items

And added two new syntax kind:

* MACRO_ITEMS
* MACRO_STMTS

Co-authored-by: Edwin Cheng <[email protected]>

4 files changed, 72 insertions, 10 deletions

diff --git a/crates/ra_parser/src/grammar.rs b/crates/ra_parser/src/grammar.rs
index f8ed1299a..e1762633e 100644
--- a/crates/ra_parser/src/grammar.rs
+++ b/crates/ra_parser/src/grammar.rs
@@ -49,6 +49,27 @@ pub(crate) fn root(p: &mut Parser) {
     m.complete(p, SOURCE_FILE);
 }
 
+pub(crate) fn macro_items(p: &mut Parser) {
+    let m = p.start();
+    items::mod_contents(p, false);
+    m.complete(p, MACRO_ITEMS);
+}
+
+pub(crate) fn macro_stmts(p: &mut Parser) {
+    let m = p.start();
+
+    while !p.at(EOF) {
+        if p.current() == SEMI {
+            p.bump();
+            continue;
+        }
+
+        expressions::stmt(p, expressions::StmtWithSemi::Optional);
+    }
+
+    m.complete(p, MACRO_STMTS);
+}
+
 pub(crate) fn path(p: &mut Parser) {
     paths::type_path(p);
 }
@@ -66,6 +87,11 @@ pub(crate) fn pattern(p: &mut Parser) {
 }
 
 pub(crate) fn stmt(p: &mut Parser, with_semi: bool) {
+    let with_semi = match with_semi {
+        true => expressions::StmtWithSemi::Yes,
+        false => expressions::StmtWithSemi::No,
+    };
+
     expressions::stmt(p, with_semi)
 }
 
diff --git a/crates/ra_parser/src/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs
index 06f2b45b1..8df9035e9 100644
--- a/crates/ra_parser/src/grammar/expressions.rs
+++ b/crates/ra_parser/src/grammar/expressions.rs
@@ -4,6 +4,12 @@ pub(crate) use self::atom::match_arm_list;
 pub(super) use self::atom::{literal, LITERAL_FIRST};
 use super::*;
 
+pub(super) enum StmtWithSemi {
+    Yes,
+    No,
+    Optional,
+}
+
 const EXPR_FIRST: TokenSet = LHS_FIRST;
 
 pub(super) fn expr(p: &mut Parser) -> BlockLike {
@@ -48,7 +54,7 @@ fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool {
     }
 }
 
-pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
+pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi) {
     // test block_items
     // fn a() { fn b() {} }
     let m = p.start();
@@ -111,13 +117,23 @@ pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
         //     }
         //     test!{}
         // }
-        if with_semi {
-            if blocklike.is_block() {
-                p.eat(SEMI);
-            } else {
-                p.expect(SEMI);
+
+        match with_semi {
+            StmtWithSemi::Yes => {
+                if blocklike.is_block() {
+                    p.eat(SEMI);
+                } else {
+                    p.expect(SEMI);
+                }
+            }
+            StmtWithSemi::No => {}
+            StmtWithSemi::Optional => {
+                if p.at(SEMI) {
+                    p.eat(SEMI);
+                }
             }
         }
+
         m.complete(p, EXPR_STMT);
     }
 
@@ -128,7 +144,7 @@ pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
     //     let c = 92;
     //     let d: i32 = 92;
     // }
-    fn let_stmt(p: &mut Parser, m: Marker, with_semi: bool) {
+    fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) {
         assert!(p.at(LET_KW));
         p.bump();
         patterns::pattern(p);
@@ -139,8 +155,16 @@ pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
             expressions::expr(p);
         }
 
-        if with_semi {
-            p.expect(SEMI);
+        match with_semi {
+            StmtWithSemi::Yes => {
+                p.expect(SEMI);
+            }
+            StmtWithSemi::No => {}
+            StmtWithSemi::Optional => {
+                if p.at(SEMI) {
+                    p.eat(SEMI);
+                }
+            }
         }
         m.complete(p, LET_STMT);
     }
@@ -160,7 +184,7 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
             continue;
         }
 
-        stmt(p, true)
+        stmt(p, StmtWithSemi::Yes)
     }
 }
 
diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs
index 11b5b9a75..0ea942b6e 100644
--- a/crates/ra_parser/src/lib.rs
+++ b/crates/ra_parser/src/lib.rs
@@ -98,6 +98,14 @@ pub fn parse_item(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink)
     parse_from_tokens(token_source, tree_sink, grammar::item);
 }
 
+pub fn parse_macro_items(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    parse_from_tokens(token_source, tree_sink, grammar::macro_items);
+}
+
+pub fn parse_macro_stmts(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    parse_from_tokens(token_source, tree_sink, grammar::macro_stmts);
+}
+
 /// A parsing function for a specific braced-block.
 pub struct Reparser(fn(&mut parser::Parser));
 
diff --git a/crates/ra_parser/src/syntax_kind/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs
index 498b0e164..6f984aea1 100644
--- a/crates/ra_parser/src/syntax_kind/generated.rs
+++ b/crates/ra_parser/src/syntax_kind/generated.rs
@@ -233,6 +233,8 @@ pub enum SyntaxKind {
     ARG_LIST,
     TYPE_BOUND,
     TYPE_BOUND_LIST,
+    MACRO_ITEMS,
+    MACRO_STMTS,
     // Technical kind so that we can cast from u16 safely
     #[doc(hidden)]
     __LAST,
@@ -592,6 +594,8 @@ impl SyntaxKind {
             ARG_LIST => &SyntaxInfo { name: "ARG_LIST" },
             TYPE_BOUND => &SyntaxInfo { name: "TYPE_BOUND" },
             TYPE_BOUND_LIST => &SyntaxInfo { name: "TYPE_BOUND_LIST" },
+            MACRO_ITEMS => &SyntaxInfo { name: "MACRO_ITEMS" },
+            MACRO_STMTS => &SyntaxInfo { name: "MACRO_STMTS" },
             TOMBSTONE => &SyntaxInfo { name: "TOMBSTONE" },
             EOF => &SyntaxInfo { name: "EOF" },
             __LAST => &SyntaxInfo { name: "__LAST" },