aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax
diff options
context:
space:
mode:
authorbors[bot] <bors[bot]@users.noreply.github.com>2019-02-24 10:15:43 +0000
committerbors[bot] <bors[bot]@users.noreply.github.com>2019-02-24 10:15:43 +0000
commitf6f160391db945a0dcc2f73b38926d6919f7c566 (patch)
tree060450b70c09357615f261d8acd032a647615dd7 /crates/ra_syntax
parentc5e74cebdcbade069c0e1e81e298ab7d729e4cd5 (diff)
parent81bca78349afb9e15994f46401da0cfabfba04a1 (diff)
Merge #885
885: Parse token trees directy r=matklad a=matklad This takes advantage of the recent macro refactoring to directly parse token stream into a syntax tree. Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_syntax')
-rw-r--r--crates/ra_syntax/src/lib.rs2
-rw-r--r--crates/ra_syntax/src/parsing.rs12
-rw-r--r--crates/ra_syntax/src/parsing/reparsing.rs8
-rw-r--r--crates/ra_syntax/src/parsing/text_token_source.rs (renamed from crates/ra_syntax/src/parsing/input.rs)10
-rw-r--r--crates/ra_syntax/src/parsing/text_tree_sink.rs (renamed from crates/ra_syntax/src/parsing/builder.rs)40
-rw-r--r--crates/ra_syntax/src/syntax_node.rs51
6 files changed, 80 insertions, 43 deletions
diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs
index e7d402446..7334d53ef 100644
--- a/crates/ra_syntax/src/lib.rs
+++ b/crates/ra_syntax/src/lib.rs
@@ -36,7 +36,7 @@ pub use crate::{
36 ast::AstNode, 36 ast::AstNode,
37 syntax_error::{SyntaxError, SyntaxErrorKind, Location}, 37 syntax_error::{SyntaxError, SyntaxErrorKind, Location},
38 syntax_text::SyntaxText, 38 syntax_text::SyntaxText,
39 syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc}, 39 syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc, SyntaxTreeBuilder},
40 ptr::{SyntaxNodePtr, AstPtr}, 40 ptr::{SyntaxNodePtr, AstPtr},
41 parsing::{tokenize, Token}, 41 parsing::{tokenize, Token},
42}; 42};
diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs
index cf573801c..ad5668a65 100644
--- a/crates/ra_syntax/src/parsing.rs
+++ b/crates/ra_syntax/src/parsing.rs
@@ -2,17 +2,13 @@
2//! incremental reparsing. 2//! incremental reparsing.
3 3
4mod lexer; 4mod lexer;
5mod input; 5mod text_token_source;
6mod builder; 6mod text_tree_sink;
7mod reparsing; 7mod reparsing;
8 8
9use crate::{ 9use crate::{
10 SyntaxError, 10 SyntaxError,
11 syntax_node::GreenNode, 11 syntax_node::GreenNode,
12 parsing::{
13 builder::TreeBuilder,
14 input::ParserInput,
15 },
16}; 12};
17 13
18pub use self::lexer::{tokenize, Token}; 14pub use self::lexer::{tokenize, Token};
@@ -21,8 +17,8 @@ pub(crate) use self::reparsing::incremental_reparse;
21 17
22pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) { 18pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
23 let tokens = tokenize(&text); 19 let tokens = tokenize(&text);
24 let token_source = ParserInput::new(text, &tokens); 20 let token_source = text_token_source::TextTokenSource::new(text, &tokens);
25 let mut tree_sink = TreeBuilder::new(text, &tokens); 21 let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens);
26 ra_parser::parse(&token_source, &mut tree_sink); 22 ra_parser::parse(&token_source, &mut tree_sink);
27 tree_sink.finish() 23 tree_sink.finish()
28} 24}
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index 19d8adcfb..ba77a3b6c 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -14,8 +14,8 @@ use crate::{
14 algo, 14 algo,
15 syntax_node::{GreenNode, SyntaxNode}, 15 syntax_node::{GreenNode, SyntaxNode},
16 parsing::{ 16 parsing::{
17 input::ParserInput, 17 text_token_source::TextTokenSource,
18 builder::TreeBuilder, 18 text_tree_sink::TextTreeSink,
19 lexer::{tokenize, Token}, 19 lexer::{tokenize, Token},
20 } 20 }
21}; 21};
@@ -68,8 +68,8 @@ fn reparse_block<'node>(
68 if !is_balanced(&tokens) { 68 if !is_balanced(&tokens) {
69 return None; 69 return None;
70 } 70 }
71 let token_source = ParserInput::new(&text, &tokens); 71 let token_source = TextTokenSource::new(&text, &tokens);
72 let mut tree_sink = TreeBuilder::new(&text, &tokens); 72 let mut tree_sink = TextTreeSink::new(&text, &tokens);
73 reparser.parse(&token_source, &mut tree_sink); 73 reparser.parse(&token_source, &mut tree_sink);
74 let (green, new_errors) = tree_sink.finish(); 74 let (green, new_errors) = tree_sink.finish();
75 Some((node, green, new_errors)) 75 Some((node, green, new_errors))
diff --git a/crates/ra_syntax/src/parsing/input.rs b/crates/ra_syntax/src/parsing/text_token_source.rs
index 31c6a3b9b..a6277f66f 100644
--- a/crates/ra_syntax/src/parsing/input.rs
+++ b/crates/ra_syntax/src/parsing/text_token_source.rs
@@ -5,7 +5,7 @@ use crate::{
5 parsing::lexer::Token, 5 parsing::lexer::Token,
6}; 6};
7 7
8pub(crate) struct ParserInput<'t> { 8pub(crate) struct TextTokenSource<'t> {
9 text: &'t str, 9 text: &'t str,
10 /// start position of each token(expect whitespace and comment) 10 /// start position of each token(expect whitespace and comment)
11 /// ```non-rust 11 /// ```non-rust
@@ -25,7 +25,7 @@ pub(crate) struct ParserInput<'t> {
25 tokens: Vec<Token>, 25 tokens: Vec<Token>,
26} 26}
27 27
28impl<'t> TokenSource for ParserInput<'t> { 28impl<'t> TokenSource for TextTokenSource<'t> {
29 fn token_kind(&self, pos: usize) -> SyntaxKind { 29 fn token_kind(&self, pos: usize) -> SyntaxKind {
30 if !(pos < self.tokens.len()) { 30 if !(pos < self.tokens.len()) {
31 return EOF; 31 return EOF;
@@ -48,9 +48,9 @@ impl<'t> TokenSource for ParserInput<'t> {
48 } 48 }
49} 49}
50 50
51impl<'t> ParserInput<'t> { 51impl<'t> TextTokenSource<'t> {
52 /// Generate input from tokens(expect comment and whitespace). 52 /// Generate input from tokens(expect comment and whitespace).
53 pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { 53 pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> {
54 let mut tokens = Vec::new(); 54 let mut tokens = Vec::new();
55 let mut start_offsets = Vec::new(); 55 let mut start_offsets = Vec::new();
56 let mut len = 0.into(); 56 let mut len = 0.into();
@@ -62,6 +62,6 @@ impl<'t> ParserInput<'t> {
62 len += token.len; 62 len += token.len;
63 } 63 }
64 64
65 ParserInput { text, start_offsets, tokens } 65 TextTokenSource { text, start_offsets, tokens }
66 } 66 }
67} 67}
diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs
index cfe3139b8..961a91d41 100644
--- a/crates/ra_syntax/src/parsing/builder.rs
+++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs
@@ -1,26 +1,24 @@
1use std::mem; 1use std::mem;
2 2
3use ra_parser::{TreeSink, ParseError}; 3use ra_parser::{TreeSink, ParseError};
4use rowan::GreenNodeBuilder;
5 4
6use crate::{ 5use crate::{
7 SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, 6 SmolStr, SyntaxError, TextUnit, TextRange, SyntaxTreeBuilder,
8 SyntaxKind::{self, *}, 7 SyntaxKind::{self, *},
9 parsing::Token, 8 parsing::Token,
10 syntax_node::{GreenNode, RaTypes}, 9 syntax_node::GreenNode,
11}; 10};
12 11
13/// Bridges the parser with our specific syntax tree representation. 12/// Bridges the parser with our specific syntax tree representation.
14/// 13///
15/// `TreeBuilder` also handles attachment of trivia (whitespace) to nodes. 14/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
16pub(crate) struct TreeBuilder<'a> { 15pub(crate) struct TextTreeSink<'a> {
17 text: &'a str, 16 text: &'a str,
18 tokens: &'a [Token], 17 tokens: &'a [Token],
19 text_pos: TextUnit, 18 text_pos: TextUnit,
20 token_pos: usize, 19 token_pos: usize,
21 state: State, 20 state: State,
22 errors: Vec<SyntaxError>, 21 inner: SyntaxTreeBuilder,
23 inner: GreenNodeBuilder<RaTypes>,
24} 22}
25 23
26enum State { 24enum State {
@@ -29,11 +27,11 @@ enum State {
29 PendingFinish, 27 PendingFinish,
30} 28}
31 29
32impl<'a> TreeSink for TreeBuilder<'a> { 30impl<'a> TreeSink for TextTreeSink<'a> {
33 fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { 31 fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
34 match mem::replace(&mut self.state, State::Normal) { 32 match mem::replace(&mut self.state, State::Normal) {
35 State::PendingStart => unreachable!(), 33 State::PendingStart => unreachable!(),
36 State::PendingFinish => self.inner.finish_internal(), 34 State::PendingFinish => self.inner.finish_branch(),
37 State::Normal => (), 35 State::Normal => (),
38 } 36 }
39 self.eat_trivias(); 37 self.eat_trivias();
@@ -48,12 +46,12 @@ impl<'a> TreeSink for TreeBuilder<'a> {
48 fn start_branch(&mut self, kind: SyntaxKind) { 46 fn start_branch(&mut self, kind: SyntaxKind) {
49 match mem::replace(&mut self.state, State::Normal) { 47 match mem::replace(&mut self.state, State::Normal) {
50 State::PendingStart => { 48 State::PendingStart => {
51 self.inner.start_internal(kind); 49 self.inner.start_branch(kind);
52 // No need to attach trivias to previous node: there is no 50 // No need to attach trivias to previous node: there is no
53 // previous node. 51 // previous node.
54 return; 52 return;
55 } 53 }
56 State::PendingFinish => self.inner.finish_internal(), 54 State::PendingFinish => self.inner.finish_branch(),
57 State::Normal => (), 55 State::Normal => (),
58 } 56 }
59 57
@@ -73,34 +71,32 @@ impl<'a> TreeSink for TreeBuilder<'a> {
73 n_attached_trivias(kind, leading_trivias) 71 n_attached_trivias(kind, leading_trivias)
74 }; 72 };
75 self.eat_n_trivias(n_trivias - n_attached_trivias); 73 self.eat_n_trivias(n_trivias - n_attached_trivias);
76 self.inner.start_internal(kind); 74 self.inner.start_branch(kind);
77 self.eat_n_trivias(n_attached_trivias); 75 self.eat_n_trivias(n_attached_trivias);
78 } 76 }
79 77
80 fn finish_branch(&mut self) { 78 fn finish_branch(&mut self) {
81 match mem::replace(&mut self.state, State::PendingFinish) { 79 match mem::replace(&mut self.state, State::PendingFinish) {
82 State::PendingStart => unreachable!(), 80 State::PendingStart => unreachable!(),
83 State::PendingFinish => self.inner.finish_internal(), 81 State::PendingFinish => self.inner.finish_branch(),
84 State::Normal => (), 82 State::Normal => (),
85 } 83 }
86 } 84 }
87 85
88 fn error(&mut self, error: ParseError) { 86 fn error(&mut self, error: ParseError) {
89 let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos); 87 self.inner.error(error, self.text_pos)
90 self.errors.push(error)
91 } 88 }
92} 89}
93 90
94impl<'a> TreeBuilder<'a> { 91impl<'a> TextTreeSink<'a> {
95 pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> { 92 pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> {
96 TreeBuilder { 93 TextTreeSink {
97 text, 94 text,
98 tokens, 95 tokens,
99 text_pos: 0.into(), 96 text_pos: 0.into(),
100 token_pos: 0, 97 token_pos: 0,
101 state: State::PendingStart, 98 state: State::PendingStart,
102 errors: Vec::new(), 99 inner: SyntaxTreeBuilder::default(),
103 inner: GreenNodeBuilder::new(),
104 } 100 }
105 } 101 }
106 102
@@ -108,12 +104,12 @@ impl<'a> TreeBuilder<'a> {
108 match mem::replace(&mut self.state, State::Normal) { 104 match mem::replace(&mut self.state, State::Normal) {
109 State::PendingFinish => { 105 State::PendingFinish => {
110 self.eat_trivias(); 106 self.eat_trivias();
111 self.inner.finish_internal() 107 self.inner.finish_branch()
112 } 108 }
113 State::PendingStart | State::Normal => unreachable!(), 109 State::PendingStart | State::Normal => unreachable!(),
114 } 110 }
115 111
116 (self.inner.finish(), self.errors) 112 self.inner.finish_raw()
117 } 113 }
118 114
119 fn eat_trivias(&mut self) { 115 fn eat_trivias(&mut self) {
diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs
index 4d54ae614..e5b4cdb11 100644
--- a/crates/ra_syntax/src/syntax_node.rs
+++ b/crates/ra_syntax/src/syntax_node.rs
@@ -11,11 +11,12 @@ use std::{
11 borrow::Borrow, 11 borrow::Borrow,
12}; 12};
13 13
14use rowan::{Types, TransparentNewType}; 14use ra_parser::ParseError;
15use rowan::{Types, TransparentNewType, GreenNodeBuilder};
15 16
16use crate::{ 17use crate::{
17 SmolStr, SyntaxKind, TextRange, SyntaxText, SourceFile, AstNode, 18 SmolStr, SyntaxKind, TextUnit, TextRange, SyntaxText, SourceFile, AstNode,
18 syntax_error::SyntaxError, 19 syntax_error::{SyntaxError, SyntaxErrorKind},
19}; 20};
20 21
21pub use rowan::WalkEvent; 22pub use rowan::WalkEvent;
@@ -276,3 +277,47 @@ fn has_short_text(kind: SyntaxKind) -> bool {
276 _ => false, 277 _ => false,
277 } 278 }
278} 279}
280
281pub struct SyntaxTreeBuilder {
282 errors: Vec<SyntaxError>,
283 inner: GreenNodeBuilder<RaTypes>,
284}
285
286impl Default for SyntaxTreeBuilder {
287 fn default() -> SyntaxTreeBuilder {
288 SyntaxTreeBuilder { errors: Vec::new(), inner: GreenNodeBuilder::new() }
289 }
290}
291
292impl SyntaxTreeBuilder {
293 pub(crate) fn finish_raw(self) -> (GreenNode, Vec<SyntaxError>) {
294 let green = self.inner.finish();
295 (green, self.errors)
296 }
297
298 pub fn finish(self) -> TreeArc<SyntaxNode> {
299 let (green, errors) = self.finish_raw();
300 let node = SyntaxNode::new(green, errors);
301 if cfg!(debug_assertions) {
302 crate::validation::validate_block_structure(&node);
303 }
304 node
305 }
306
307 pub fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) {
308 self.inner.leaf(kind, text)
309 }
310
311 pub fn start_branch(&mut self, kind: SyntaxKind) {
312 self.inner.start_internal(kind)
313 }
314
315 pub fn finish_branch(&mut self) {
316 self.inner.finish_internal()
317 }
318
319 pub fn error(&mut self, error: ParseError, text_pos: TextUnit) {
320 let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), text_pos);
321 self.errors.push(error)
322 }
323}