From 415c891d641fa305e7ddbbbcc78db990dd5d3564 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 29 Jul 2018 15:16:07 +0300 Subject: Reorganize --- src/parser/event.rs | 33 +++++++++++++++++++++++++++------ src/parser/grammar/mod.rs | 13 ++++++++----- src/parser/input.rs | 10 ++++++---- src/parser/mod.rs | 26 ++++++++------------------ src/parser/parser/imp.rs | 3 +-- src/parser/parser/mod.rs | 3 +-- 6 files changed, 51 insertions(+), 37 deletions(-) (limited to 'src/parser') diff --git a/src/parser/event.rs b/src/parser/event.rs index 83039c664..a8d503b3d 100644 --- a/src/parser/event.rs +++ b/src/parser/event.rs @@ -1,8 +1,29 @@ +//! This module provides a way to construct a `File`. +//! It is intended to be completely decoupled from the +//! parser, so as to allow to evolve the tree representation +//! and the parser algorithm independently. +//! +//! The `Sink` trait is the bridge between the parser and the +//! tree builder: the parser produces a stream of events like +//! `start node`, `finish node`, and `FileBuilder` converts +//! this stream to a real tree. use { - Sink, SyntaxKind, Token, - syntax_kinds::TOMBSTONE, + TextUnit, + SyntaxKind::{self, TOMBSTONE}, + lexer::Token, }; -use super::is_insignificant; + +pub(crate) trait Sink { + type Tree; + + fn new(text: String) -> Self; + + fn leaf(&mut self, kind: SyntaxKind, len: TextUnit); + fn start_internal(&mut self, kind: SyntaxKind); + fn finish_internal(&mut self); + fn error(&mut self, err: String); + fn finish(self) -> Self::Tree; +} /// `Parser` produces a flat list of `Event`s. /// They are converted to a tree-structure in @@ -67,7 +88,7 @@ pub(crate) enum Event { }, } -pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec) { +pub(super) fn process(builder: &mut impl Sink, tokens: &[Token], events: Vec) { let mut idx = 0; let mut holes = Vec::new(); @@ -111,7 +132,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec) &Event::Finish => { while idx < tokens.len() { let token = tokens[idx]; - if is_insignificant(token.kind) { + if token.kind.is_trivia() { idx += 1; builder.leaf(token.kind, token.len); } else { @@ -128,7 +149,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec) // this should be done in a sensible manner instead loop { let token = tokens[idx]; - if !is_insignificant(token.kind) { + if !token.kind.is_trivia() { break; } builder.leaf(token.kind, token.len); diff --git a/src/parser/grammar/mod.rs b/src/parser/grammar/mod.rs index 23216452f..085e62d56 100644 --- a/src/parser/grammar/mod.rs +++ b/src/parser/grammar/mod.rs @@ -21,11 +21,6 @@ //! After adding a new inline-test, run `cargo collect-tests` to extract //! it as a standalone text-fixture into `tests/data/parser/inline`, and //! run `cargo test` once to create the "gold" value. -use parser::parser::Parser; -use parser::token_set::TokenSet; -use SyntaxKind; -use syntax_kinds::*; - mod items; mod attributes; mod expressions; @@ -34,6 +29,14 @@ mod patterns; mod paths; mod type_params; +use { + SyntaxKind::{self, *}, + parser::{ + parser::Parser, + token_set::TokenSet + } +}; + pub(crate) fn file(p: &mut Parser) { let file = p.start(); p.eat(SHEBANG); diff --git a/src/parser/input.rs b/src/parser/input.rs index 9b400b959..052981fbc 100644 --- a/src/parser/input.rs +++ b/src/parser/input.rs @@ -1,6 +1,8 @@ -use {SyntaxKind, TextRange, TextUnit, Token}; -use syntax_kinds::EOF; -use super::is_insignificant; +use { + SyntaxKind, TextRange, TextUnit, + SyntaxKind::EOF, + lexer::Token, +}; use std::ops::{Add, AddAssign}; @@ -16,7 +18,7 @@ impl<'t> ParserInput<'t> { let mut start_offsets = Vec::new(); let mut len = 0.into(); for &token in raw_tokens.iter() { - if !is_insignificant(token.kind) { + if !token.kind.is_trivia() { tokens.push(token); start_offsets.push(len); } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b7d5e5832..e72ab05af 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5,18 +5,16 @@ mod input; mod event; mod grammar; -use std::sync::Arc; use { - Token, - yellow::SyntaxNode, - syntax_kinds::* + lexer::Token, + parser::event::{process} }; -use GreenBuilder; -use parser::event::process; + +pub(crate) use self::event::Sink; /// Parse a sequence of tokens into the representative node tree -pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode { +pub(crate) fn parse(text: String, tokens: &[Token]) -> S::Tree { let events = { let input = input::ParserInput::new(&text, tokens); let parser_impl = parser::imp::ParserImpl::new(&input); @@ -24,15 +22,7 @@ pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode { grammar::file(&mut parser); parser.0.into_events() }; - let mut builder = GreenBuilder::new(text); - process(&mut builder, tokens, events); - let (green, errors) = builder.finish(); - SyntaxNode::new(Arc::new(green), errors) -} - -fn is_insignificant(kind: SyntaxKind) -> bool { - match kind { - WHITESPACE | COMMENT => true, - _ => false, - } + let mut sink = S::new(text); + process(&mut sink, tokens, events); + sink.finish() } diff --git a/src/parser/parser/imp.rs b/src/parser/parser/imp.rs index f2641c388..38237ac06 100644 --- a/src/parser/parser/imp.rs +++ b/src/parser/parser/imp.rs @@ -1,8 +1,7 @@ use parser::input::{InputPosition, ParserInput}; use parser::event::Event; -use SyntaxKind; -use syntax_kinds::{EOF, TOMBSTONE}; +use SyntaxKind::{self, EOF, TOMBSTONE}; /// Implementation details of `Parser`, extracted /// to a separate struct in order not to pollute diff --git a/src/parser/parser/mod.rs b/src/parser/parser/mod.rs index 58f820738..0930ff9e4 100644 --- a/src/parser/parser/mod.rs +++ b/src/parser/parser/mod.rs @@ -1,5 +1,4 @@ -use SyntaxKind; -use syntax_kinds::ERROR; +use SyntaxKind::{self, ERROR}; pub(super) mod imp; use self::imp::ParserImpl; -- cgit v1.2.3