From 0c1cb981820c55127c3c09d93868814a1df98246 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 23 Feb 2019 16:07:29 +0300 Subject: rename --- crates/ra_syntax/src/parsing/builder.rs | 170 ---------------------- crates/ra_syntax/src/parsing/input.rs | 67 --------- crates/ra_syntax/src/parsing/reparsing.rs | 8 +- crates/ra_syntax/src/parsing/text_token_source.rs | 67 +++++++++ crates/ra_syntax/src/parsing/text_tree_sink.rs | 170 ++++++++++++++++++++++ 5 files changed, 241 insertions(+), 241 deletions(-) delete mode 100644 crates/ra_syntax/src/parsing/builder.rs delete mode 100644 crates/ra_syntax/src/parsing/input.rs create mode 100644 crates/ra_syntax/src/parsing/text_token_source.rs create mode 100644 crates/ra_syntax/src/parsing/text_tree_sink.rs (limited to 'crates/ra_syntax/src/parsing') diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/builder.rs deleted file mode 100644 index cfe3139b8..000000000 --- a/crates/ra_syntax/src/parsing/builder.rs +++ /dev/null @@ -1,170 +0,0 @@ -use std::mem; - -use ra_parser::{TreeSink, ParseError}; -use rowan::GreenNodeBuilder; - -use crate::{ - SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, - SyntaxKind::{self, *}, - parsing::Token, - syntax_node::{GreenNode, RaTypes}, -}; - -/// Bridges the parser with our specific syntax tree representation. -/// -/// `TreeBuilder` also handles attachment of trivia (whitespace) to nodes. -pub(crate) struct TreeBuilder<'a> { - text: &'a str, - tokens: &'a [Token], - text_pos: TextUnit, - token_pos: usize, - state: State, - errors: Vec, - inner: GreenNodeBuilder, -} - -enum State { - PendingStart, - Normal, - PendingFinish, -} - -impl<'a> TreeSink for TreeBuilder<'a> { - fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_internal(), - State::Normal => (), - } - self.eat_trivias(); - let n_tokens = n_tokens as usize; - let len = self.tokens[self.token_pos..self.token_pos + n_tokens] - .iter() - .map(|it| it.len) - .sum::(); - self.do_leaf(kind, len, n_tokens); - } - - fn start_branch(&mut self, kind: SyntaxKind) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingStart => { - self.inner.start_internal(kind); - // No need to attach trivias to previous node: there is no - // previous node. - return; - } - State::PendingFinish => self.inner.finish_internal(), - State::Normal => (), - } - - let n_trivias = - self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); - let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; - let mut trivia_end = - self.text_pos + leading_trivias.iter().map(|it| it.len).sum::(); - - let n_attached_trivias = { - let leading_trivias = leading_trivias.iter().rev().map(|it| { - let next_end = trivia_end - it.len; - let range = TextRange::from_to(next_end, trivia_end); - trivia_end = next_end; - (it.kind, &self.text[range]) - }); - n_attached_trivias(kind, leading_trivias) - }; - self.eat_n_trivias(n_trivias - n_attached_trivias); - self.inner.start_internal(kind); - self.eat_n_trivias(n_attached_trivias); - } - - fn finish_branch(&mut self) { - match mem::replace(&mut self.state, State::PendingFinish) { - State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_internal(), - State::Normal => (), - } - } - - fn error(&mut self, error: ParseError) { - let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos); - self.errors.push(error) - } -} - -impl<'a> TreeBuilder<'a> { - pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> { - TreeBuilder { - text, - tokens, - text_pos: 0.into(), - token_pos: 0, - state: State::PendingStart, - errors: Vec::new(), - inner: GreenNodeBuilder::new(), - } - } - - pub(super) fn finish(mut self) -> (GreenNode, Vec) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingFinish => { - self.eat_trivias(); - self.inner.finish_internal() - } - State::PendingStart | State::Normal => unreachable!(), - } - - (self.inner.finish(), self.errors) - } - - fn eat_trivias(&mut self) { - while let Some(&token) = self.tokens.get(self.token_pos) { - if !token.kind.is_trivia() { - break; - } - self.do_leaf(token.kind, token.len, 1); - } - } - - fn eat_n_trivias(&mut self, n: usize) { - for _ in 0..n { - let token = self.tokens[self.token_pos]; - assert!(token.kind.is_trivia()); - self.do_leaf(token.kind, token.len, 1); - } - } - - fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { - let range = TextRange::offset_len(self.text_pos, len); - let text: SmolStr = self.text[range].into(); - self.text_pos += len; - self.token_pos += n_tokens; - self.inner.leaf(kind, text); - } -} - -fn n_attached_trivias<'a>( - kind: SyntaxKind, - trivias: impl Iterator, -) -> usize { - match kind { - CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF - | MODULE | NAMED_FIELD_DEF => { - let mut res = 0; - for (i, (kind, text)) in trivias.enumerate() { - match kind { - WHITESPACE => { - if text.contains("\n\n") { - break; - } - } - COMMENT => { - res = i + 1; - } - _ => (), - } - } - res - } - _ => 0, - } -} diff --git a/crates/ra_syntax/src/parsing/input.rs b/crates/ra_syntax/src/parsing/input.rs deleted file mode 100644 index 31c6a3b9b..000000000 --- a/crates/ra_syntax/src/parsing/input.rs +++ /dev/null @@ -1,67 +0,0 @@ -use ra_parser::TokenSource; - -use crate::{ - SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, - parsing::lexer::Token, -}; - -pub(crate) struct ParserInput<'t> { - text: &'t str, - /// start position of each token(expect whitespace and comment) - /// ```non-rust - /// struct Foo; - /// ^------^--- - /// | | ^- - /// 0 7 10 - /// ``` - /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` - start_offsets: Vec, - /// non-whitespace/comment tokens - /// ```non-rust - /// struct Foo {} - /// ^^^^^^ ^^^ ^^ - /// ``` - /// tokens: `[struct, Foo, {, }]` - tokens: Vec, -} - -impl<'t> TokenSource for ParserInput<'t> { - fn token_kind(&self, pos: usize) -> SyntaxKind { - if !(pos < self.tokens.len()) { - return EOF; - } - self.tokens[pos].kind - } - fn is_token_joint_to_next(&self, pos: usize) -> bool { - if !(pos + 1 < self.tokens.len()) { - return true; - } - self.start_offsets[pos] + self.tokens[pos].len == self.start_offsets[pos + 1] - } - fn is_keyword(&self, pos: usize, kw: &str) -> bool { - if !(pos < self.tokens.len()) { - return false; - } - let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len); - - self.text[range] == *kw - } -} - -impl<'t> ParserInput<'t> { - /// Generate input from tokens(expect comment and whitespace). - pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { - let mut tokens = Vec::new(); - let mut start_offsets = Vec::new(); - let mut len = 0.into(); - for &token in raw_tokens.iter() { - if !token.kind.is_trivia() { - tokens.push(token); - start_offsets.push(len); - } - len += token.len; - } - - ParserInput { text, start_offsets, tokens } - } -} diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 19d8adcfb..ba77a3b6c 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -14,8 +14,8 @@ use crate::{ algo, syntax_node::{GreenNode, SyntaxNode}, parsing::{ - input::ParserInput, - builder::TreeBuilder, + text_token_source::TextTokenSource, + text_tree_sink::TextTreeSink, lexer::{tokenize, Token}, } }; @@ -68,8 +68,8 @@ fn reparse_block<'node>( if !is_balanced(&tokens) { return None; } - let token_source = ParserInput::new(&text, &tokens); - let mut tree_sink = TreeBuilder::new(&text, &tokens); + let token_source = TextTokenSource::new(&text, &tokens); + let mut tree_sink = TextTreeSink::new(&text, &tokens); reparser.parse(&token_source, &mut tree_sink); let (green, new_errors) = tree_sink.finish(); Some((node, green, new_errors)) diff --git a/crates/ra_syntax/src/parsing/text_token_source.rs b/crates/ra_syntax/src/parsing/text_token_source.rs new file mode 100644 index 000000000..a6277f66f --- /dev/null +++ b/crates/ra_syntax/src/parsing/text_token_source.rs @@ -0,0 +1,67 @@ +use ra_parser::TokenSource; + +use crate::{ + SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, + parsing::lexer::Token, +}; + +pub(crate) struct TextTokenSource<'t> { + text: &'t str, + /// start position of each token(expect whitespace and comment) + /// ```non-rust + /// struct Foo; + /// ^------^--- + /// | | ^- + /// 0 7 10 + /// ``` + /// (token, start_offset): `[(struct, 0), (Foo, 7), (;, 10)]` + start_offsets: Vec, + /// non-whitespace/comment tokens + /// ```non-rust + /// struct Foo {} + /// ^^^^^^ ^^^ ^^ + /// ``` + /// tokens: `[struct, Foo, {, }]` + tokens: Vec, +} + +impl<'t> TokenSource for TextTokenSource<'t> { + fn token_kind(&self, pos: usize) -> SyntaxKind { + if !(pos < self.tokens.len()) { + return EOF; + } + self.tokens[pos].kind + } + fn is_token_joint_to_next(&self, pos: usize) -> bool { + if !(pos + 1 < self.tokens.len()) { + return true; + } + self.start_offsets[pos] + self.tokens[pos].len == self.start_offsets[pos + 1] + } + fn is_keyword(&self, pos: usize, kw: &str) -> bool { + if !(pos < self.tokens.len()) { + return false; + } + let range = TextRange::offset_len(self.start_offsets[pos], self.tokens[pos].len); + + self.text[range] == *kw + } +} + +impl<'t> TextTokenSource<'t> { + /// Generate input from tokens(expect comment and whitespace). + pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { + let mut tokens = Vec::new(); + let mut start_offsets = Vec::new(); + let mut len = 0.into(); + for &token in raw_tokens.iter() { + if !token.kind.is_trivia() { + tokens.push(token); + start_offsets.push(len); + } + len += token.len; + } + + TextTokenSource { text, start_offsets, tokens } + } +} diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs new file mode 100644 index 000000000..8c1d78deb --- /dev/null +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs @@ -0,0 +1,170 @@ +use std::mem; + +use ra_parser::{TreeSink, ParseError}; +use rowan::GreenNodeBuilder; + +use crate::{ + SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, + SyntaxKind::{self, *}, + parsing::Token, + syntax_node::{GreenNode, RaTypes}, +}; + +/// Bridges the parser with our specific syntax tree representation. +/// +/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. +pub(crate) struct TextTreeSink<'a> { + text: &'a str, + tokens: &'a [Token], + text_pos: TextUnit, + token_pos: usize, + state: State, + errors: Vec, + inner: GreenNodeBuilder, +} + +enum State { + PendingStart, + Normal, + PendingFinish, +} + +impl<'a> TreeSink for TextTreeSink<'a> { + fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingStart => unreachable!(), + State::PendingFinish => self.inner.finish_internal(), + State::Normal => (), + } + self.eat_trivias(); + let n_tokens = n_tokens as usize; + let len = self.tokens[self.token_pos..self.token_pos + n_tokens] + .iter() + .map(|it| it.len) + .sum::(); + self.do_leaf(kind, len, n_tokens); + } + + fn start_branch(&mut self, kind: SyntaxKind) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingStart => { + self.inner.start_internal(kind); + // No need to attach trivias to previous node: there is no + // previous node. + return; + } + State::PendingFinish => self.inner.finish_internal(), + State::Normal => (), + } + + let n_trivias = + self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); + let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; + let mut trivia_end = + self.text_pos + leading_trivias.iter().map(|it| it.len).sum::(); + + let n_attached_trivias = { + let leading_trivias = leading_trivias.iter().rev().map(|it| { + let next_end = trivia_end - it.len; + let range = TextRange::from_to(next_end, trivia_end); + trivia_end = next_end; + (it.kind, &self.text[range]) + }); + n_attached_trivias(kind, leading_trivias) + }; + self.eat_n_trivias(n_trivias - n_attached_trivias); + self.inner.start_internal(kind); + self.eat_n_trivias(n_attached_trivias); + } + + fn finish_branch(&mut self) { + match mem::replace(&mut self.state, State::PendingFinish) { + State::PendingStart => unreachable!(), + State::PendingFinish => self.inner.finish_internal(), + State::Normal => (), + } + } + + fn error(&mut self, error: ParseError) { + let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos); + self.errors.push(error) + } +} + +impl<'a> TextTreeSink<'a> { + pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> { + TextTreeSink { + text, + tokens, + text_pos: 0.into(), + token_pos: 0, + state: State::PendingStart, + errors: Vec::new(), + inner: GreenNodeBuilder::new(), + } + } + + pub(super) fn finish(mut self) -> (GreenNode, Vec) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingFinish => { + self.eat_trivias(); + self.inner.finish_internal() + } + State::PendingStart | State::Normal => unreachable!(), + } + + (self.inner.finish(), self.errors) + } + + fn eat_trivias(&mut self) { + while let Some(&token) = self.tokens.get(self.token_pos) { + if !token.kind.is_trivia() { + break; + } + self.do_leaf(token.kind, token.len, 1); + } + } + + fn eat_n_trivias(&mut self, n: usize) { + for _ in 0..n { + let token = self.tokens[self.token_pos]; + assert!(token.kind.is_trivia()); + self.do_leaf(token.kind, token.len, 1); + } + } + + fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { + let range = TextRange::offset_len(self.text_pos, len); + let text: SmolStr = self.text[range].into(); + self.text_pos += len; + self.token_pos += n_tokens; + self.inner.leaf(kind, text); + } +} + +fn n_attached_trivias<'a>( + kind: SyntaxKind, + trivias: impl Iterator, +) -> usize { + match kind { + CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF + | MODULE | NAMED_FIELD_DEF => { + let mut res = 0; + for (i, (kind, text)) in trivias.enumerate() { + match kind { + WHITESPACE => { + if text.contains("\n\n") { + break; + } + } + COMMENT => { + res = i + 1; + } + _ => (), + } + } + res + } + _ => 0, + } +} -- cgit v1.2.3 From f078f7adc8ac0ffae07462d736083807c98c0483 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 23 Feb 2019 16:55:01 +0300 Subject: introduce tree builder --- crates/ra_syntax/src/parsing/text_tree_sink.rs | 28 +++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) (limited to 'crates/ra_syntax/src/parsing') diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs index 8c1d78deb..961a91d41 100644 --- a/crates/ra_syntax/src/parsing/text_tree_sink.rs +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs @@ -1,13 +1,12 @@ use std::mem; use ra_parser::{TreeSink, ParseError}; -use rowan::GreenNodeBuilder; use crate::{ - SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, + SmolStr, SyntaxError, TextUnit, TextRange, SyntaxTreeBuilder, SyntaxKind::{self, *}, parsing::Token, - syntax_node::{GreenNode, RaTypes}, + syntax_node::GreenNode, }; /// Bridges the parser with our specific syntax tree representation. @@ -19,8 +18,7 @@ pub(crate) struct TextTreeSink<'a> { text_pos: TextUnit, token_pos: usize, state: State, - errors: Vec, - inner: GreenNodeBuilder, + inner: SyntaxTreeBuilder, } enum State { @@ -33,7 +31,7 @@ impl<'a> TreeSink for TextTreeSink<'a> { fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_internal(), + State::PendingFinish => self.inner.finish_branch(), State::Normal => (), } self.eat_trivias(); @@ -48,12 +46,12 @@ impl<'a> TreeSink for TextTreeSink<'a> { fn start_branch(&mut self, kind: SyntaxKind) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => { - self.inner.start_internal(kind); + self.inner.start_branch(kind); // No need to attach trivias to previous node: there is no // previous node. return; } - State::PendingFinish => self.inner.finish_internal(), + State::PendingFinish => self.inner.finish_branch(), State::Normal => (), } @@ -73,21 +71,20 @@ impl<'a> TreeSink for TextTreeSink<'a> { n_attached_trivias(kind, leading_trivias) }; self.eat_n_trivias(n_trivias - n_attached_trivias); - self.inner.start_internal(kind); + self.inner.start_branch(kind); self.eat_n_trivias(n_attached_trivias); } fn finish_branch(&mut self) { match mem::replace(&mut self.state, State::PendingFinish) { State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_internal(), + State::PendingFinish => self.inner.finish_branch(), State::Normal => (), } } fn error(&mut self, error: ParseError) { - let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos); - self.errors.push(error) + self.inner.error(error, self.text_pos) } } @@ -99,8 +96,7 @@ impl<'a> TextTreeSink<'a> { text_pos: 0.into(), token_pos: 0, state: State::PendingStart, - errors: Vec::new(), - inner: GreenNodeBuilder::new(), + inner: SyntaxTreeBuilder::default(), } } @@ -108,12 +104,12 @@ impl<'a> TextTreeSink<'a> { match mem::replace(&mut self.state, State::Normal) { State::PendingFinish => { self.eat_trivias(); - self.inner.finish_internal() + self.inner.finish_branch() } State::PendingStart | State::Normal => unreachable!(), } - (self.inner.finish(), self.errors) + self.inner.finish_raw() } fn eat_trivias(&mut self) { -- cgit v1.2.3