Merge #885

885: Parse token trees directy r=matklad a=matklad

This takes advantage of the recent macro refactoring to directly parse token stream into a syntax tree.

Co-authored-by: Aleksey Kladov <[email protected]>

3 files changed, 27 insertions, 31 deletions

diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index 19d8adcfb..ba77a3b6c 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -14,8 +14,8 @@ use crate::{
     algo,
     syntax_node::{GreenNode, SyntaxNode},
     parsing::{
-        input::ParserInput,
-        builder::TreeBuilder,
+        text_token_source::TextTokenSource,
+        text_tree_sink::TextTreeSink,
         lexer::{tokenize, Token},
     }
 };
@@ -68,8 +68,8 @@ fn reparse_block<'node>(
     if !is_balanced(&tokens) {
         return None;
     }
-    let token_source = ParserInput::new(&text, &tokens);
-    let mut tree_sink = TreeBuilder::new(&text, &tokens);
+    let token_source = TextTokenSource::new(&text, &tokens);
+    let mut tree_sink = TextTreeSink::new(&text, &tokens);
     reparser.parse(&token_source, &mut tree_sink);
     let (green, new_errors) = tree_sink.finish();
     Some((node, green, new_errors))
diff --git a/crates/ra_syntax/src/parsing/input.rs b/crates/ra_syntax/src/parsing/text_token_source.rs
index 31c6a3b9b..a6277f66f 100644
--- a/crates/ra_syntax/src/parsing/input.rs
+++ b/crates/ra_syntax/src/parsing/text_token_source.rs
@@ -5,7 +5,7 @@ use crate::{
     parsing::lexer::Token,
 };
 
-pub(crate) struct ParserInput<'t> {
+pub(crate) struct TextTokenSource<'t> {
     text: &'t str,
     /// start position of each token(expect whitespace and comment)
     /// ```non-rust
@@ -25,7 +25,7 @@ pub(crate) struct ParserInput<'t> {
     tokens: Vec<Token>,
 }
 
-impl<'t> TokenSource for ParserInput<'t> {
+impl<'t> TokenSource for TextTokenSource<'t> {
     fn token_kind(&self, pos: usize) -> SyntaxKind {
         if !(pos < self.tokens.len()) {
             return EOF;
@@ -48,9 +48,9 @@ impl<'t> TokenSource for ParserInput<'t> {
     }
 }
 
-impl<'t> ParserInput<'t> {
+impl<'t> TextTokenSource<'t> {
     /// Generate input from tokens(expect comment and whitespace).
-    pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> {
+    pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
         let mut tokens = Vec::new();
         let mut start_offsets = Vec::new();
         let mut len = 0.into();
@@ -62,6 +62,6 @@ impl<'t> ParserInput<'t> {
             len += token.len;
         }
 
-        ParserInput { text, start_offsets, tokens }
+        TextTokenSource { text, start_offsets, tokens }
     }
 }
diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs
index cfe3139b8..961a91d41 100644
--- a/crates/ra_syntax/src/parsing/builder.rs
+++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs
@@ -1,26 +1,24 @@
 use std::mem;
 
 use ra_parser::{TreeSink, ParseError};
-use rowan::GreenNodeBuilder;
 
 use crate::{
-    SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange,
+    SmolStr, SyntaxError, TextUnit, TextRange, SyntaxTreeBuilder,
     SyntaxKind::{self, *},
     parsing::Token,
-    syntax_node::{GreenNode, RaTypes},
+    syntax_node::GreenNode,
 };
 
 /// Bridges the parser with our specific syntax tree representation.
 ///
-/// `TreeBuilder` also handles attachment of trivia (whitespace) to nodes.
-pub(crate) struct TreeBuilder<'a> {
+/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
+pub(crate) struct TextTreeSink<'a> {
     text: &'a str,
     tokens: &'a [Token],
     text_pos: TextUnit,
     token_pos: usize,
     state: State,
-    errors: Vec<SyntaxError>,
-    inner: GreenNodeBuilder<RaTypes>,
+    inner: SyntaxTreeBuilder,
 }
 
 enum State {
@@ -29,11 +27,11 @@ enum State {
     PendingFinish,
 }
 
-impl<'a> TreeSink for TreeBuilder<'a> {
+impl<'a> TreeSink for TextTreeSink<'a> {
     fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingStart => unreachable!(),
-            State::PendingFinish => self.inner.finish_internal(),
+            State::PendingFinish => self.inner.finish_branch(),
             State::Normal => (),
         }
         self.eat_trivias();
@@ -48,12 +46,12 @@ impl<'a> TreeSink for TreeBuilder<'a> {
     fn start_branch(&mut self, kind: SyntaxKind) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingStart => {
-                self.inner.start_internal(kind);
+                self.inner.start_branch(kind);
                 // No need to attach trivias to previous node: there is no
                 // previous node.
                 return;
             }
-            State::PendingFinish => self.inner.finish_internal(),
+            State::PendingFinish => self.inner.finish_branch(),
             State::Normal => (),
         }
 
@@ -73,34 +71,32 @@ impl<'a> TreeSink for TreeBuilder<'a> {
             n_attached_trivias(kind, leading_trivias)
         };
         self.eat_n_trivias(n_trivias - n_attached_trivias);
-        self.inner.start_internal(kind);
+        self.inner.start_branch(kind);
         self.eat_n_trivias(n_attached_trivias);
     }
 
     fn finish_branch(&mut self) {
         match mem::replace(&mut self.state, State::PendingFinish) {
             State::PendingStart => unreachable!(),
-            State::PendingFinish => self.inner.finish_internal(),
+            State::PendingFinish => self.inner.finish_branch(),
             State::Normal => (),
         }
     }
 
     fn error(&mut self, error: ParseError) {
-        let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos);
-        self.errors.push(error)
+        self.inner.error(error, self.text_pos)
     }
 }
 
-impl<'a> TreeBuilder<'a> {
-    pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> {
-        TreeBuilder {
+impl<'a> TextTreeSink<'a> {
+    pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> {
+        TextTreeSink {
             text,
             tokens,
             text_pos: 0.into(),
             token_pos: 0,
             state: State::PendingStart,
-            errors: Vec::new(),
-            inner: GreenNodeBuilder::new(),
+            inner: SyntaxTreeBuilder::default(),
         }
     }
 
@@ -108,12 +104,12 @@ impl<'a> TreeBuilder<'a> {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingFinish => {
                 self.eat_trivias();
-                self.inner.finish_internal()
+                self.inner.finish_branch()
             }
             State::PendingStart | State::Normal => unreachable!(),
         }
 
-        (self.inner.finish(), self.errors)
+        self.inner.finish_raw()
     }
 
     fn eat_trivias(&mut self) {