6 files changed, 51 insertions, 37 deletions

diff --git a/src/parser/event.rs b/src/parser/event.rs
index 83039c664..a8d503b3d 100644
--- a/src/parser/event.rs
+++ b/src/parser/event.rs
@@ -1,8 +1,29 @@
+//! This module provides a way to construct a `File`.
+//! It is intended to be completely decoupled from the
+//! parser, so as to allow to evolve the tree representation
+//! and the parser algorithm independently.
+//!
+//! The `Sink` trait is the bridge between the parser and the
+//! tree builder: the parser produces a stream of events like
+//! `start node`, `finish node`, and `FileBuilder` converts
+//! this stream to a real tree.
 use {
-    Sink, SyntaxKind, Token,
-    syntax_kinds::TOMBSTONE,
+    TextUnit,
+    SyntaxKind::{self, TOMBSTONE},
+    lexer::Token,
 };
-use super::is_insignificant;
+
+pub(crate) trait Sink {
+    type Tree;
+
+    fn new(text: String) -> Self;
+
+    fn leaf(&mut self, kind: SyntaxKind, len: TextUnit);
+    fn start_internal(&mut self, kind: SyntaxKind);
+    fn finish_internal(&mut self);
+    fn error(&mut self, err: String);
+    fn finish(self) -> Self::Tree;
+}
 
 /// `Parser` produces a flat list of `Event`s.
 /// They are converted to a tree-structure in
@@ -67,7 +88,7 @@ pub(crate) enum Event {
     },
 }
 
-pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>) {
+pub(super) fn process(builder: &mut impl Sink, tokens: &[Token], events: Vec<Event>) {
     let mut idx = 0;
 
     let mut holes = Vec::new();
@@ -111,7 +132,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>)
             &Event::Finish => {
                 while idx < tokens.len() {
                     let token = tokens[idx];
-                    if is_insignificant(token.kind) {
+                    if token.kind.is_trivia() {
                         idx += 1;
                         builder.leaf(token.kind, token.len);
                     } else {
@@ -128,7 +149,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>)
                 // this should be done in a sensible manner instead
                 loop {
                     let token = tokens[idx];
-                    if !is_insignificant(token.kind) {
+                    if !token.kind.is_trivia() {
                         break;
                     }
                     builder.leaf(token.kind, token.len);
diff --git a/src/parser/grammar/mod.rs b/src/parser/grammar/mod.rs
index 23216452f..085e62d56 100644
--- a/src/parser/grammar/mod.rs
+++ b/src/parser/grammar/mod.rs
@@ -21,11 +21,6 @@
 //! After adding a new inline-test, run `cargo collect-tests` to extract
 //! it as a standalone text-fixture into `tests/data/parser/inline`, and
 //! run `cargo test` once to create the "gold" value.
-use parser::parser::Parser;
-use parser::token_set::TokenSet;
-use SyntaxKind;
-use syntax_kinds::*;
-
 mod items;
 mod attributes;
 mod expressions;
@@ -34,6 +29,14 @@ mod patterns;
 mod paths;
 mod type_params;
 
+use {
+    SyntaxKind::{self, *},
+    parser::{
+        parser::Parser,
+        token_set::TokenSet
+    }
+};
+
 pub(crate) fn file(p: &mut Parser) {
     let file = p.start();
     p.eat(SHEBANG);
diff --git a/src/parser/input.rs b/src/parser/input.rs
index 9b400b959..052981fbc 100644
--- a/src/parser/input.rs
+++ b/src/parser/input.rs
@@ -1,6 +1,8 @@
-use {SyntaxKind, TextRange, TextUnit, Token};
-use syntax_kinds::EOF;
-use super::is_insignificant;
+use {
+    SyntaxKind, TextRange, TextUnit,
+    SyntaxKind::EOF,
+    lexer::Token,
+};
 
 use std::ops::{Add, AddAssign};
 
@@ -16,7 +18,7 @@ impl<'t> ParserInput<'t> {
         let mut start_offsets = Vec::new();
         let mut len = 0.into();
         for &token in raw_tokens.iter() {
-            if !is_insignificant(token.kind) {
+            if !token.kind.is_trivia() {
                 tokens.push(token);
                 start_offsets.push(len);
             }
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
index b7d5e5832..e72ab05af 100644
--- a/src/parser/mod.rs
+++ b/src/parser/mod.rs
@@ -5,18 +5,16 @@ mod input;
 mod event;
 mod grammar;
 
-use std::sync::Arc;
 use {
-    Token,
-    yellow::SyntaxNode,
-    syntax_kinds::*
+    lexer::Token,
+    parser::event::{process}
 };
-use GreenBuilder;
-use parser::event::process;
+
+pub(crate) use self::event::Sink;
 
 
 /// Parse a sequence of tokens into the representative node tree
-pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode {
+pub(crate) fn parse<S: Sink>(text: String, tokens: &[Token]) -> S::Tree {
     let events = {
         let input = input::ParserInput::new(&text, tokens);
         let parser_impl = parser::imp::ParserImpl::new(&input);
@@ -24,15 +22,7 @@ pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode {
         grammar::file(&mut parser);
         parser.0.into_events()
     };
-    let mut builder = GreenBuilder::new(text);
-    process(&mut builder, tokens, events);
-    let (green, errors) = builder.finish();
-    SyntaxNode::new(Arc::new(green), errors)
-}
-
-fn is_insignificant(kind: SyntaxKind) -> bool {
-    match kind {
-        WHITESPACE | COMMENT => true,
-        _ => false,
-    }
+    let mut sink = S::new(text);
+    process(&mut sink, tokens, events);
+    sink.finish()
 }
diff --git a/src/parser/parser/imp.rs b/src/parser/parser/imp.rs
index f2641c388..38237ac06 100644
--- a/src/parser/parser/imp.rs
+++ b/src/parser/parser/imp.rs
@@ -1,8 +1,7 @@
 use parser::input::{InputPosition, ParserInput};
 use parser::event::Event;
 
-use SyntaxKind;
-use syntax_kinds::{EOF, TOMBSTONE};
+use SyntaxKind::{self, EOF, TOMBSTONE};
 
 /// Implementation details of `Parser`, extracted
 /// to a separate struct in order not to pollute
diff --git a/src/parser/parser/mod.rs b/src/parser/parser/mod.rs
index 58f820738..0930ff9e4 100644
--- a/src/parser/parser/mod.rs
+++ b/src/parser/parser/mod.rs
@@ -1,5 +1,4 @@
-use SyntaxKind;
-use syntax_kinds::ERROR;
+use SyntaxKind::{self, ERROR};
 
 pub(super) mod imp;
 use self::imp::ParserImpl;