Merge #885

885: Parse token trees directy r=matklad a=matklad

This takes advantage of the recent macro refactoring to directly parse token stream into a syntax tree.

Co-authored-by: Aleksey Kladov <[email protected]>

10 files changed, 288 insertions, 47 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 5912659d4..e697de588 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1066,6 +1066,7 @@ dependencies = [
 name = "ra_mbe"
 version = "0.1.0"
 dependencies = [
+ "ra_parser 0.1.0",
  "ra_syntax 0.1.0",
  "ra_tt 0.1.0",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/crates/ra_mbe/Cargo.toml b/crates/ra_mbe/Cargo.toml
index e7b8660e7..6e785f570 100644
--- a/crates/ra_mbe/Cargo.toml
+++ b/crates/ra_mbe/Cargo.toml
@@ -6,6 +6,7 @@ authors = ["rust-analyzer developers"]
 
 [dependencies]
 ra_syntax = { path = "../ra_syntax" }
+ra_parser = { path = "../ra_parser" }
 tt = { path = "../ra_tt", package = "ra_tt" }
 
 rustc-hash = "1.0.0"
diff --git a/crates/ra_mbe/src/lib.rs b/crates/ra_mbe/src/lib.rs
index cdca3cafb..907402f5f 100644
--- a/crates/ra_mbe/src/lib.rs
+++ b/crates/ra_mbe/src/lib.rs
@@ -24,7 +24,7 @@ use ra_syntax::SmolStr;
 
 pub use tt::{Delimiter, Punct};
 
-pub use crate::syntax_bridge::ast_to_token_tree;
+pub use crate::syntax_bridge::{ast_to_token_tree, token_tree_to_ast_item_list};
 
 /// This struct contains AST for a single `macro_rules` definition. What might
 /// be very confusing is that AST has almost exactly the same shape as
@@ -164,14 +164,18 @@ impl_froms!(TokenTree: Leaf, Subtree);
         crate::MacroRules::parse(&definition_tt).unwrap()
     }
 
-    fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) {
+    fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
         let source_file = ast::SourceFile::parse(invocation);
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
 
         let (invocation_tt, _) = ast_to_token_tree(macro_invocation.token_tree().unwrap()).unwrap();
 
-        let expanded = rules.expand(&invocation_tt).unwrap();
+        rules.expand(&invocation_tt).unwrap()
+    }
+
+    fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) {
+        let expanded = expand(rules, invocation);
         assert_eq!(expanded.to_string(), expansion);
     }
 
@@ -268,4 +272,57 @@ impl_froms!(TokenTree: Leaf, Subtree);
         assert_expansion(&rules, "foo! { Foo,# Bar }", "struct Foo ; struct Bar ;");
     }
 
+    #[test]
+    fn expand_to_item_list() {
+        let rules = create_rules(
+            "
+            macro_rules! structs {
+                ($($i:ident),*) => {
+                    $(struct $i { field: u32 } )*
+                }
+            }
+            ",
+        );
+        let expansion = expand(&rules, "structs!(Foo, Bar)");
+        let tree = token_tree_to_ast_item_list(&expansion);
+        assert_eq!(
+            tree.syntax().debug_dump().trim(),
+            r#"
+SOURCE_FILE@[0; 40)
+  STRUCT_DEF@[0; 20)
+    STRUCT_KW@[0; 6)
+    NAME@[6; 9)
+      IDENT@[6; 9) "Foo"
+    NAMED_FIELD_DEF_LIST@[9; 20)
+      L_CURLY@[9; 10)
+      NAMED_FIELD_DEF@[10; 19)
+        NAME@[10; 15)
+          IDENT@[10; 15) "field"
+        COLON@[15; 16)
+        PATH_TYPE@[16; 19)
+          PATH@[16; 19)
+            PATH_SEGMENT@[16; 19)
+              NAME_REF@[16; 19)
+                IDENT@[16; 19) "u32"
+      R_CURLY@[19; 20)
+  STRUCT_DEF@[20; 40)
+    STRUCT_KW@[20; 26)
+    NAME@[26; 29)
+      IDENT@[26; 29) "Bar"
+    NAMED_FIELD_DEF_LIST@[29; 40)
+      L_CURLY@[29; 30)
+      NAMED_FIELD_DEF@[30; 39)
+        NAME@[30; 35)
+          IDENT@[30; 35) "field"
+        COLON@[35; 36)
+        PATH_TYPE@[36; 39)
+          PATH@[36; 39)
+            PATH_SEGMENT@[36; 39)
+              NAME_REF@[36; 39)
+                IDENT@[36; 39) "u32"
+      R_CURLY@[39; 40)"#
+                .trim()
+        );
+    }
+
 }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 848c785f8..c1472bbe5 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -1,5 +1,6 @@
+use ra_parser::{TokenSource, TreeSink, ParseError};
 use ra_syntax::{
-    AstNode, SyntaxNode, TextRange,
+    AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc,
     ast, SyntaxKind::*, TextUnit
 };
 
@@ -19,6 +20,15 @@ pub fn ast_to_token_tree(ast: &ast::TokenTree) -> Option<(tt::Subtree, TokenMap)
     Some((tt, token_map))
 }
 
+/// Parses the token tree (result of macro expansion) as a sequence of items
+pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> {
+    let token_source = TtTokenSource::new(tt);
+    let mut tree_sink = TtTreeSink::new(&token_source.tokens);
+    ra_parser::parse(&token_source, &mut tree_sink);
+    let syntax = tree_sink.inner.finish();
+    ast::SourceFile::cast(&syntax).unwrap().to_owned()
+}
+
 impl TokenMap {
     pub fn relative_range_of(&self, tt: tt::TokenId) -> Option<TextRange> {
         let idx = tt.0 as usize;
@@ -84,3 +94,138 @@ fn convert_tt(
     let res = tt::Subtree { delimiter, token_trees };
     Some(res)
 }
+
+struct TtTokenSource {
+    tokens: Vec<TtToken>,
+}
+
+struct TtToken {
+    kind: SyntaxKind,
+    is_joint_to_next: bool,
+    text: SmolStr,
+}
+
+impl TtTokenSource {
+    fn new(tt: &tt::Subtree) -> TtTokenSource {
+        let mut res = TtTokenSource { tokens: Vec::new() };
+        res.convert_subtree(tt);
+        res
+    }
+    fn convert_subtree(&mut self, sub: &tt::Subtree) {
+        self.push_delim(sub.delimiter, false);
+        sub.token_trees.iter().for_each(|tt| self.convert_tt(tt));
+        self.push_delim(sub.delimiter, true)
+    }
+    fn convert_tt(&mut self, tt: &tt::TokenTree) {
+        match tt {
+            tt::TokenTree::Leaf(leaf) => self.convert_leaf(leaf),
+            tt::TokenTree::Subtree(sub) => self.convert_subtree(sub),
+        }
+    }
+    fn convert_leaf(&mut self, leaf: &tt::Leaf) {
+        let tok = match leaf {
+            tt::Leaf::Literal(l) => TtToken {
+                kind: SyntaxKind::INT_NUMBER, // FIXME
+                is_joint_to_next: false,
+                text: l.text.clone(),
+            },
+            tt::Leaf::Punct(p) => {
+                let kind = match p.char {
+                    // lexer may produce combpund tokens for these ones
+                    '.' => DOT,
+                    ':' => COLON,
+                    '=' => EQ,
+                    '!' => EXCL,
+                    '-' => MINUS,
+                    c => SyntaxKind::from_char(c).unwrap(),
+                };
+                let text = {
+                    let mut buf = [0u8; 4];
+                    let s: &str = p.char.encode_utf8(&mut buf);
+                    SmolStr::new(s)
+                };
+                TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
+            }
+            tt::Leaf::Ident(ident) => {
+                let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT);
+                TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
+            }
+        };
+        self.tokens.push(tok)
+    }
+    fn push_delim(&mut self, d: tt::Delimiter, closing: bool) {
+        let (kinds, texts) = match d {
+            tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"),
+            tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"),
+            tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"),
+            tt::Delimiter::None => return,
+        };
+        let idx = closing as usize;
+        let kind = kinds[idx];
+        let text = &texts[idx..texts.len() - (1 - idx)];
+        let tok = TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) };
+        self.tokens.push(tok)
+    }
+}
+
+impl TokenSource for TtTokenSource {
+    fn token_kind(&self, pos: usize) -> SyntaxKind {
+        if let Some(tok) = self.tokens.get(pos) {
+            tok.kind
+        } else {
+            SyntaxKind::EOF
+        }
+    }
+    fn is_token_joint_to_next(&self, pos: usize) -> bool {
+        self.tokens[pos].is_joint_to_next
+    }
+    fn is_keyword(&self, pos: usize, kw: &str) -> bool {
+        self.tokens[pos].text == *kw
+    }
+}
+
+#[derive(Default)]
+struct TtTreeSink<'a> {
+    buf: String,
+    tokens: &'a [TtToken],
+    text_pos: TextUnit,
+    token_pos: usize,
+    inner: SyntaxTreeBuilder,
+}
+
+impl<'a> TtTreeSink<'a> {
+    fn new(tokens: &'a [TtToken]) -> TtTreeSink {
+        TtTreeSink {
+            buf: String::new(),
+            tokens,
+            text_pos: 0.into(),
+            token_pos: 0,
+            inner: SyntaxTreeBuilder::default(),
+        }
+    }
+}
+
+impl<'a> TreeSink for TtTreeSink<'a> {
+    fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
+        for _ in 0..n_tokens {
+            self.buf += self.tokens[self.token_pos].text.as_str();
+            self.token_pos += 1;
+        }
+        self.text_pos += TextUnit::of_str(&self.buf);
+        let text = SmolStr::new(self.buf.as_str());
+        self.buf.clear();
+        self.inner.leaf(kind, text)
+    }
+
+    fn start_branch(&mut self, kind: SyntaxKind) {
+        self.inner.start_branch(kind);
+    }
+
+    fn finish_branch(&mut self) {
+        self.inner.finish_branch();
+    }
+
+    fn error(&mut self, error: ParseError) {
+        self.inner.error(error, self.text_pos)
+    }
+}
diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs
index e7d402446..7334d53ef 100644
--- a/crates/ra_syntax/src/lib.rs
+++ b/crates/ra_syntax/src/lib.rs
@@ -36,7 +36,7 @@ pub use crate::{
     ast::AstNode,
     syntax_error::{SyntaxError, SyntaxErrorKind, Location},
     syntax_text::SyntaxText,
-    syntax_node::{Direction,  SyntaxNode, WalkEvent, TreeArc},
+    syntax_node::{Direction,  SyntaxNode, WalkEvent, TreeArc, SyntaxTreeBuilder},
     ptr::{SyntaxNodePtr, AstPtr},
     parsing::{tokenize, Token},
 };
diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs
index cf573801c..ad5668a65 100644
--- a/crates/ra_syntax/src/parsing.rs
+++ b/crates/ra_syntax/src/parsing.rs
@@ -2,17 +2,13 @@
 //! incremental reparsing.
 
 mod lexer;
-mod input;
-mod builder;
+mod text_token_source;
+mod text_tree_sink;
 mod reparsing;
 
 use crate::{
     SyntaxError,
     syntax_node::GreenNode,
-    parsing::{
-        builder::TreeBuilder,
-        input::ParserInput,
-    },
 };
 
 pub use self::lexer::{tokenize, Token};
@@ -21,8 +17,8 @@ pub(crate) use self::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
     let tokens = tokenize(&text);
-    let token_source = ParserInput::new(text, &tokens);
-    let mut tree_sink = TreeBuilder::new(text, &tokens);
+    let token_source = text_token_source::TextTokenSource::new(text, &tokens);
+    let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens);
     ra_parser::parse(&token_source, &mut tree_sink);
     tree_sink.finish()
 }
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index 19d8adcfb..ba77a3b6c 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -14,8 +14,8 @@ use crate::{
     algo,
     syntax_node::{GreenNode, SyntaxNode},
     parsing::{
-        input::ParserInput,
-        builder::TreeBuilder,
+        text_token_source::TextTokenSource,
+        text_tree_sink::TextTreeSink,
         lexer::{tokenize, Token},
     }
 };
@@ -68,8 +68,8 @@ fn reparse_block<'node>(
     if !is_balanced(&tokens) {
         return None;
     }
-    let token_source = ParserInput::new(&text, &tokens);
-    let mut tree_sink = TreeBuilder::new(&text, &tokens);
+    let token_source = TextTokenSource::new(&text, &tokens);
+    let mut tree_sink = TextTreeSink::new(&text, &tokens);
     reparser.parse(&token_source, &mut tree_sink);
     let (green, new_errors) = tree_sink.finish();
     Some((node, green, new_errors))
diff --git a/crates/ra_syntax/src/parsing/input.rs b/crates/ra_syntax/src/parsing/text_token_source.rs
index 31c6a3b9b..a6277f66f 100644
--- a/crates/ra_syntax/src/parsing/input.rs
+++ b/crates/ra_syntax/src/parsing/text_token_source.rs
@@ -5,7 +5,7 @@ use crate::{
     parsing::lexer::Token,
 };
 
-pub(crate) struct ParserInput<'t> {
+pub(crate) struct TextTokenSource<'t> {
     text: &'t str,
     /// start position of each token(expect whitespace and comment)
     /// ```non-rust
@@ -25,7 +25,7 @@ pub(crate) struct ParserInput<'t> {
     tokens: Vec<Token>,
 }
 
-impl<'t> TokenSource for ParserInput<'t> {
+impl<'t> TokenSource for TextTokenSource<'t> {
     fn token_kind(&self, pos: usize) -> SyntaxKind {
         if !(pos < self.tokens.len()) {
             return EOF;
@@ -48,9 +48,9 @@ impl<'t> TokenSource for ParserInput<'t> {
     }
 }
 
-impl<'t> ParserInput<'t> {
+impl<'t> TextTokenSource<'t> {
     /// Generate input from tokens(expect comment and whitespace).
-    pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> {
+    pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
         let mut tokens = Vec::new();
         let mut start_offsets = Vec::new();
         let mut len = 0.into();
@@ -62,6 +62,6 @@ impl<'t> ParserInput<'t> {
             len += token.len;
         }
 
-        ParserInput { text, start_offsets, tokens }
+        TextTokenSource { text, start_offsets, tokens }
     }
 }
diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs
index cfe3139b8..961a91d41 100644
--- a/crates/ra_syntax/src/parsing/builder.rs
+++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs
@@ -1,26 +1,24 @@
 use std::mem;
 
 use ra_parser::{TreeSink, ParseError};
-use rowan::GreenNodeBuilder;
 
 use crate::{
-    SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange,
+    SmolStr, SyntaxError, TextUnit, TextRange, SyntaxTreeBuilder,
     SyntaxKind::{self, *},
     parsing::Token,
-    syntax_node::{GreenNode, RaTypes},
+    syntax_node::GreenNode,
 };
 
 /// Bridges the parser with our specific syntax tree representation.
 ///
-/// `TreeBuilder` also handles attachment of trivia (whitespace) to nodes.
-pub(crate) struct TreeBuilder<'a> {
+/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
+pub(crate) struct TextTreeSink<'a> {
     text: &'a str,
     tokens: &'a [Token],
     text_pos: TextUnit,
     token_pos: usize,
     state: State,
-    errors: Vec<SyntaxError>,
-    inner: GreenNodeBuilder<RaTypes>,
+    inner: SyntaxTreeBuilder,
 }
 
 enum State {
@@ -29,11 +27,11 @@ enum State {
     PendingFinish,
 }
 
-impl<'a> TreeSink for TreeBuilder<'a> {
+impl<'a> TreeSink for TextTreeSink<'a> {
     fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingStart => unreachable!(),
-            State::PendingFinish => self.inner.finish_internal(),
+            State::PendingFinish => self.inner.finish_branch(),
             State::Normal => (),
         }
         self.eat_trivias();
@@ -48,12 +46,12 @@ impl<'a> TreeSink for TreeBuilder<'a> {
     fn start_branch(&mut self, kind: SyntaxKind) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingStart => {
-                self.inner.start_internal(kind);
+                self.inner.start_branch(kind);
                 // No need to attach trivias to previous node: there is no
                 // previous node.
                 return;
             }
-            State::PendingFinish => self.inner.finish_internal(),
+            State::PendingFinish => self.inner.finish_branch(),
             State::Normal => (),
         }
 
@@ -73,34 +71,32 @@ impl<'a> TreeSink for TreeBuilder<'a> {
             n_attached_trivias(kind, leading_trivias)
         };
         self.eat_n_trivias(n_trivias - n_attached_trivias);
-        self.inner.start_internal(kind);
+        self.inner.start_branch(kind);
         self.eat_n_trivias(n_attached_trivias);
     }
 
     fn finish_branch(&mut self) {
         match mem::replace(&mut self.state, State::PendingFinish) {
             State::PendingStart => unreachable!(),
-            State::PendingFinish => self.inner.finish_internal(),
+            State::PendingFinish => self.inner.finish_branch(),
             State::Normal => (),
         }
     }
 
     fn error(&mut self, error: ParseError) {
-        let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos);
-        self.errors.push(error)
+        self.inner.error(error, self.text_pos)
     }
 }
 
-impl<'a> TreeBuilder<'a> {
-    pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> {
-        TreeBuilder {
+impl<'a> TextTreeSink<'a> {
+    pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> {
+        TextTreeSink {
             text,
             tokens,
             text_pos: 0.into(),
             token_pos: 0,
             state: State::PendingStart,
-            errors: Vec::new(),
-            inner: GreenNodeBuilder::new(),
+            inner: SyntaxTreeBuilder::default(),
         }
     }
 
@@ -108,12 +104,12 @@ impl<'a> TreeBuilder<'a> {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingFinish => {
                 self.eat_trivias();
-                self.inner.finish_internal()
+                self.inner.finish_branch()
             }
             State::PendingStart | State::Normal => unreachable!(),
         }
 
-        (self.inner.finish(), self.errors)
+        self.inner.finish_raw()
     }
 
     fn eat_trivias(&mut self) {
diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs
index 4d54ae614..e5b4cdb11 100644
--- a/crates/ra_syntax/src/syntax_node.rs
+++ b/crates/ra_syntax/src/syntax_node.rs
@@ -11,11 +11,12 @@ use std::{
     borrow::Borrow,
 };
 
-use rowan::{Types, TransparentNewType};
+use ra_parser::ParseError;
+use rowan::{Types, TransparentNewType, GreenNodeBuilder};
 
 use crate::{
-    SmolStr, SyntaxKind, TextRange, SyntaxText, SourceFile, AstNode,
-    syntax_error::SyntaxError,
+    SmolStr, SyntaxKind, TextUnit, TextRange, SyntaxText, SourceFile, AstNode,
+    syntax_error::{SyntaxError, SyntaxErrorKind},
 };
 
 pub use rowan::WalkEvent;
@@ -276,3 +277,47 @@ fn has_short_text(kind: SyntaxKind) -> bool {
         _ => false,
     }
 }
+
+pub struct SyntaxTreeBuilder {
+    errors: Vec<SyntaxError>,
+    inner: GreenNodeBuilder<RaTypes>,
+}
+
+impl Default for SyntaxTreeBuilder {
+    fn default() -> SyntaxTreeBuilder {
+        SyntaxTreeBuilder { errors: Vec::new(), inner: GreenNodeBuilder::new() }
+    }
+}
+
+impl SyntaxTreeBuilder {
+    pub(crate) fn finish_raw(self) -> (GreenNode, Vec<SyntaxError>) {
+        let green = self.inner.finish();
+        (green, self.errors)
+    }
+
+    pub fn finish(self) -> TreeArc<SyntaxNode> {
+        let (green, errors) = self.finish_raw();
+        let node = SyntaxNode::new(green, errors);
+        if cfg!(debug_assertions) {
+            crate::validation::validate_block_structure(&node);
+        }
+        node
+    }
+
+    pub fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) {
+        self.inner.leaf(kind, text)
+    }
+
+    pub fn start_branch(&mut self, kind: SyntaxKind) {
+        self.inner.start_internal(kind)
+    }
+
+    pub fn finish_branch(&mut self) {
+        self.inner.finish_internal()
+    }
+
+    pub fn error(&mut self, error: ParseError, text_pos: TextUnit) {
+        let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), text_pos);
+        self.errors.push(error)
+    }
+}