diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-24 10:15:43 +0000 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-24 10:15:43 +0000 |
commit | f6f160391db945a0dcc2f73b38926d6919f7c566 (patch) | |
tree | 060450b70c09357615f261d8acd032a647615dd7 /crates/ra_syntax | |
parent | c5e74cebdcbade069c0e1e81e298ab7d729e4cd5 (diff) | |
parent | 81bca78349afb9e15994f46401da0cfabfba04a1 (diff) |
Merge #885
885: Parse token trees directy r=matklad a=matklad
This takes advantage of the recent macro refactoring to directly parse token stream into a syntax tree.
Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_syntax')
-rw-r--r-- | crates/ra_syntax/src/lib.rs | 2 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing.rs | 12 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/reparsing.rs | 8 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/text_token_source.rs (renamed from crates/ra_syntax/src/parsing/input.rs) | 10 | ||||
-rw-r--r-- | crates/ra_syntax/src/parsing/text_tree_sink.rs (renamed from crates/ra_syntax/src/parsing/builder.rs) | 40 | ||||
-rw-r--r-- | crates/ra_syntax/src/syntax_node.rs | 51 |
6 files changed, 80 insertions, 43 deletions
diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs index e7d402446..7334d53ef 100644 --- a/crates/ra_syntax/src/lib.rs +++ b/crates/ra_syntax/src/lib.rs | |||
@@ -36,7 +36,7 @@ pub use crate::{ | |||
36 | ast::AstNode, | 36 | ast::AstNode, |
37 | syntax_error::{SyntaxError, SyntaxErrorKind, Location}, | 37 | syntax_error::{SyntaxError, SyntaxErrorKind, Location}, |
38 | syntax_text::SyntaxText, | 38 | syntax_text::SyntaxText, |
39 | syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc}, | 39 | syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc, SyntaxTreeBuilder}, |
40 | ptr::{SyntaxNodePtr, AstPtr}, | 40 | ptr::{SyntaxNodePtr, AstPtr}, |
41 | parsing::{tokenize, Token}, | 41 | parsing::{tokenize, Token}, |
42 | }; | 42 | }; |
diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index cf573801c..ad5668a65 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs | |||
@@ -2,17 +2,13 @@ | |||
2 | //! incremental reparsing. | 2 | //! incremental reparsing. |
3 | 3 | ||
4 | mod lexer; | 4 | mod lexer; |
5 | mod input; | 5 | mod text_token_source; |
6 | mod builder; | 6 | mod text_tree_sink; |
7 | mod reparsing; | 7 | mod reparsing; |
8 | 8 | ||
9 | use crate::{ | 9 | use crate::{ |
10 | SyntaxError, | 10 | SyntaxError, |
11 | syntax_node::GreenNode, | 11 | syntax_node::GreenNode, |
12 | parsing::{ | ||
13 | builder::TreeBuilder, | ||
14 | input::ParserInput, | ||
15 | }, | ||
16 | }; | 12 | }; |
17 | 13 | ||
18 | pub use self::lexer::{tokenize, Token}; | 14 | pub use self::lexer::{tokenize, Token}; |
@@ -21,8 +17,8 @@ pub(crate) use self::reparsing::incremental_reparse; | |||
21 | 17 | ||
22 | pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) { | 18 | pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) { |
23 | let tokens = tokenize(&text); | 19 | let tokens = tokenize(&text); |
24 | let token_source = ParserInput::new(text, &tokens); | 20 | let token_source = text_token_source::TextTokenSource::new(text, &tokens); |
25 | let mut tree_sink = TreeBuilder::new(text, &tokens); | 21 | let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens); |
26 | ra_parser::parse(&token_source, &mut tree_sink); | 22 | ra_parser::parse(&token_source, &mut tree_sink); |
27 | tree_sink.finish() | 23 | tree_sink.finish() |
28 | } | 24 | } |
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 19d8adcfb..ba77a3b6c 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs | |||
@@ -14,8 +14,8 @@ use crate::{ | |||
14 | algo, | 14 | algo, |
15 | syntax_node::{GreenNode, SyntaxNode}, | 15 | syntax_node::{GreenNode, SyntaxNode}, |
16 | parsing::{ | 16 | parsing::{ |
17 | input::ParserInput, | 17 | text_token_source::TextTokenSource, |
18 | builder::TreeBuilder, | 18 | text_tree_sink::TextTreeSink, |
19 | lexer::{tokenize, Token}, | 19 | lexer::{tokenize, Token}, |
20 | } | 20 | } |
21 | }; | 21 | }; |
@@ -68,8 +68,8 @@ fn reparse_block<'node>( | |||
68 | if !is_balanced(&tokens) { | 68 | if !is_balanced(&tokens) { |
69 | return None; | 69 | return None; |
70 | } | 70 | } |
71 | let token_source = ParserInput::new(&text, &tokens); | 71 | let token_source = TextTokenSource::new(&text, &tokens); |
72 | let mut tree_sink = TreeBuilder::new(&text, &tokens); | 72 | let mut tree_sink = TextTreeSink::new(&text, &tokens); |
73 | reparser.parse(&token_source, &mut tree_sink); | 73 | reparser.parse(&token_source, &mut tree_sink); |
74 | let (green, new_errors) = tree_sink.finish(); | 74 | let (green, new_errors) = tree_sink.finish(); |
75 | Some((node, green, new_errors)) | 75 | Some((node, green, new_errors)) |
diff --git a/crates/ra_syntax/src/parsing/input.rs b/crates/ra_syntax/src/parsing/text_token_source.rs index 31c6a3b9b..a6277f66f 100644 --- a/crates/ra_syntax/src/parsing/input.rs +++ b/crates/ra_syntax/src/parsing/text_token_source.rs | |||
@@ -5,7 +5,7 @@ use crate::{ | |||
5 | parsing::lexer::Token, | 5 | parsing::lexer::Token, |
6 | }; | 6 | }; |
7 | 7 | ||
8 | pub(crate) struct ParserInput<'t> { | 8 | pub(crate) struct TextTokenSource<'t> { |
9 | text: &'t str, | 9 | text: &'t str, |
10 | /// start position of each token(expect whitespace and comment) | 10 | /// start position of each token(expect whitespace and comment) |
11 | /// ```non-rust | 11 | /// ```non-rust |
@@ -25,7 +25,7 @@ pub(crate) struct ParserInput<'t> { | |||
25 | tokens: Vec<Token>, | 25 | tokens: Vec<Token>, |
26 | } | 26 | } |
27 | 27 | ||
28 | impl<'t> TokenSource for ParserInput<'t> { | 28 | impl<'t> TokenSource for TextTokenSource<'t> { |
29 | fn token_kind(&self, pos: usize) -> SyntaxKind { | 29 | fn token_kind(&self, pos: usize) -> SyntaxKind { |
30 | if !(pos < self.tokens.len()) { | 30 | if !(pos < self.tokens.len()) { |
31 | return EOF; | 31 | return EOF; |
@@ -48,9 +48,9 @@ impl<'t> TokenSource for ParserInput<'t> { | |||
48 | } | 48 | } |
49 | } | 49 | } |
50 | 50 | ||
51 | impl<'t> ParserInput<'t> { | 51 | impl<'t> TextTokenSource<'t> { |
52 | /// Generate input from tokens(expect comment and whitespace). | 52 | /// Generate input from tokens(expect comment and whitespace). |
53 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> { | 53 | pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> TextTokenSource<'t> { |
54 | let mut tokens = Vec::new(); | 54 | let mut tokens = Vec::new(); |
55 | let mut start_offsets = Vec::new(); | 55 | let mut start_offsets = Vec::new(); |
56 | let mut len = 0.into(); | 56 | let mut len = 0.into(); |
@@ -62,6 +62,6 @@ impl<'t> ParserInput<'t> { | |||
62 | len += token.len; | 62 | len += token.len; |
63 | } | 63 | } |
64 | 64 | ||
65 | ParserInput { text, start_offsets, tokens } | 65 | TextTokenSource { text, start_offsets, tokens } |
66 | } | 66 | } |
67 | } | 67 | } |
diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs index cfe3139b8..961a91d41 100644 --- a/crates/ra_syntax/src/parsing/builder.rs +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs | |||
@@ -1,26 +1,24 @@ | |||
1 | use std::mem; | 1 | use std::mem; |
2 | 2 | ||
3 | use ra_parser::{TreeSink, ParseError}; | 3 | use ra_parser::{TreeSink, ParseError}; |
4 | use rowan::GreenNodeBuilder; | ||
5 | 4 | ||
6 | use crate::{ | 5 | use crate::{ |
7 | SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, | 6 | SmolStr, SyntaxError, TextUnit, TextRange, SyntaxTreeBuilder, |
8 | SyntaxKind::{self, *}, | 7 | SyntaxKind::{self, *}, |
9 | parsing::Token, | 8 | parsing::Token, |
10 | syntax_node::{GreenNode, RaTypes}, | 9 | syntax_node::GreenNode, |
11 | }; | 10 | }; |
12 | 11 | ||
13 | /// Bridges the parser with our specific syntax tree representation. | 12 | /// Bridges the parser with our specific syntax tree representation. |
14 | /// | 13 | /// |
15 | /// `TreeBuilder` also handles attachment of trivia (whitespace) to nodes. | 14 | /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. |
16 | pub(crate) struct TreeBuilder<'a> { | 15 | pub(crate) struct TextTreeSink<'a> { |
17 | text: &'a str, | 16 | text: &'a str, |
18 | tokens: &'a [Token], | 17 | tokens: &'a [Token], |
19 | text_pos: TextUnit, | 18 | text_pos: TextUnit, |
20 | token_pos: usize, | 19 | token_pos: usize, |
21 | state: State, | 20 | state: State, |
22 | errors: Vec<SyntaxError>, | 21 | inner: SyntaxTreeBuilder, |
23 | inner: GreenNodeBuilder<RaTypes>, | ||
24 | } | 22 | } |
25 | 23 | ||
26 | enum State { | 24 | enum State { |
@@ -29,11 +27,11 @@ enum State { | |||
29 | PendingFinish, | 27 | PendingFinish, |
30 | } | 28 | } |
31 | 29 | ||
32 | impl<'a> TreeSink for TreeBuilder<'a> { | 30 | impl<'a> TreeSink for TextTreeSink<'a> { |
33 | fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { | 31 | fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { |
34 | match mem::replace(&mut self.state, State::Normal) { | 32 | match mem::replace(&mut self.state, State::Normal) { |
35 | State::PendingStart => unreachable!(), | 33 | State::PendingStart => unreachable!(), |
36 | State::PendingFinish => self.inner.finish_internal(), | 34 | State::PendingFinish => self.inner.finish_branch(), |
37 | State::Normal => (), | 35 | State::Normal => (), |
38 | } | 36 | } |
39 | self.eat_trivias(); | 37 | self.eat_trivias(); |
@@ -48,12 +46,12 @@ impl<'a> TreeSink for TreeBuilder<'a> { | |||
48 | fn start_branch(&mut self, kind: SyntaxKind) { | 46 | fn start_branch(&mut self, kind: SyntaxKind) { |
49 | match mem::replace(&mut self.state, State::Normal) { | 47 | match mem::replace(&mut self.state, State::Normal) { |
50 | State::PendingStart => { | 48 | State::PendingStart => { |
51 | self.inner.start_internal(kind); | 49 | self.inner.start_branch(kind); |
52 | // No need to attach trivias to previous node: there is no | 50 | // No need to attach trivias to previous node: there is no |
53 | // previous node. | 51 | // previous node. |
54 | return; | 52 | return; |
55 | } | 53 | } |
56 | State::PendingFinish => self.inner.finish_internal(), | 54 | State::PendingFinish => self.inner.finish_branch(), |
57 | State::Normal => (), | 55 | State::Normal => (), |
58 | } | 56 | } |
59 | 57 | ||
@@ -73,34 +71,32 @@ impl<'a> TreeSink for TreeBuilder<'a> { | |||
73 | n_attached_trivias(kind, leading_trivias) | 71 | n_attached_trivias(kind, leading_trivias) |
74 | }; | 72 | }; |
75 | self.eat_n_trivias(n_trivias - n_attached_trivias); | 73 | self.eat_n_trivias(n_trivias - n_attached_trivias); |
76 | self.inner.start_internal(kind); | 74 | self.inner.start_branch(kind); |
77 | self.eat_n_trivias(n_attached_trivias); | 75 | self.eat_n_trivias(n_attached_trivias); |
78 | } | 76 | } |
79 | 77 | ||
80 | fn finish_branch(&mut self) { | 78 | fn finish_branch(&mut self) { |
81 | match mem::replace(&mut self.state, State::PendingFinish) { | 79 | match mem::replace(&mut self.state, State::PendingFinish) { |
82 | State::PendingStart => unreachable!(), | 80 | State::PendingStart => unreachable!(), |
83 | State::PendingFinish => self.inner.finish_internal(), | 81 | State::PendingFinish => self.inner.finish_branch(), |
84 | State::Normal => (), | 82 | State::Normal => (), |
85 | } | 83 | } |
86 | } | 84 | } |
87 | 85 | ||
88 | fn error(&mut self, error: ParseError) { | 86 | fn error(&mut self, error: ParseError) { |
89 | let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos); | 87 | self.inner.error(error, self.text_pos) |
90 | self.errors.push(error) | ||
91 | } | 88 | } |
92 | } | 89 | } |
93 | 90 | ||
94 | impl<'a> TreeBuilder<'a> { | 91 | impl<'a> TextTreeSink<'a> { |
95 | pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TreeBuilder<'a> { | 92 | pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> { |
96 | TreeBuilder { | 93 | TextTreeSink { |
97 | text, | 94 | text, |
98 | tokens, | 95 | tokens, |
99 | text_pos: 0.into(), | 96 | text_pos: 0.into(), |
100 | token_pos: 0, | 97 | token_pos: 0, |
101 | state: State::PendingStart, | 98 | state: State::PendingStart, |
102 | errors: Vec::new(), | 99 | inner: SyntaxTreeBuilder::default(), |
103 | inner: GreenNodeBuilder::new(), | ||
104 | } | 100 | } |
105 | } | 101 | } |
106 | 102 | ||
@@ -108,12 +104,12 @@ impl<'a> TreeBuilder<'a> { | |||
108 | match mem::replace(&mut self.state, State::Normal) { | 104 | match mem::replace(&mut self.state, State::Normal) { |
109 | State::PendingFinish => { | 105 | State::PendingFinish => { |
110 | self.eat_trivias(); | 106 | self.eat_trivias(); |
111 | self.inner.finish_internal() | 107 | self.inner.finish_branch() |
112 | } | 108 | } |
113 | State::PendingStart | State::Normal => unreachable!(), | 109 | State::PendingStart | State::Normal => unreachable!(), |
114 | } | 110 | } |
115 | 111 | ||
116 | (self.inner.finish(), self.errors) | 112 | self.inner.finish_raw() |
117 | } | 113 | } |
118 | 114 | ||
119 | fn eat_trivias(&mut self) { | 115 | fn eat_trivias(&mut self) { |
diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs index 4d54ae614..e5b4cdb11 100644 --- a/crates/ra_syntax/src/syntax_node.rs +++ b/crates/ra_syntax/src/syntax_node.rs | |||
@@ -11,11 +11,12 @@ use std::{ | |||
11 | borrow::Borrow, | 11 | borrow::Borrow, |
12 | }; | 12 | }; |
13 | 13 | ||
14 | use rowan::{Types, TransparentNewType}; | 14 | use ra_parser::ParseError; |
15 | use rowan::{Types, TransparentNewType, GreenNodeBuilder}; | ||
15 | 16 | ||
16 | use crate::{ | 17 | use crate::{ |
17 | SmolStr, SyntaxKind, TextRange, SyntaxText, SourceFile, AstNode, | 18 | SmolStr, SyntaxKind, TextUnit, TextRange, SyntaxText, SourceFile, AstNode, |
18 | syntax_error::SyntaxError, | 19 | syntax_error::{SyntaxError, SyntaxErrorKind}, |
19 | }; | 20 | }; |
20 | 21 | ||
21 | pub use rowan::WalkEvent; | 22 | pub use rowan::WalkEvent; |
@@ -276,3 +277,47 @@ fn has_short_text(kind: SyntaxKind) -> bool { | |||
276 | _ => false, | 277 | _ => false, |
277 | } | 278 | } |
278 | } | 279 | } |
280 | |||
281 | pub struct SyntaxTreeBuilder { | ||
282 | errors: Vec<SyntaxError>, | ||
283 | inner: GreenNodeBuilder<RaTypes>, | ||
284 | } | ||
285 | |||
286 | impl Default for SyntaxTreeBuilder { | ||
287 | fn default() -> SyntaxTreeBuilder { | ||
288 | SyntaxTreeBuilder { errors: Vec::new(), inner: GreenNodeBuilder::new() } | ||
289 | } | ||
290 | } | ||
291 | |||
292 | impl SyntaxTreeBuilder { | ||
293 | pub(crate) fn finish_raw(self) -> (GreenNode, Vec<SyntaxError>) { | ||
294 | let green = self.inner.finish(); | ||
295 | (green, self.errors) | ||
296 | } | ||
297 | |||
298 | pub fn finish(self) -> TreeArc<SyntaxNode> { | ||
299 | let (green, errors) = self.finish_raw(); | ||
300 | let node = SyntaxNode::new(green, errors); | ||
301 | if cfg!(debug_assertions) { | ||
302 | crate::validation::validate_block_structure(&node); | ||
303 | } | ||
304 | node | ||
305 | } | ||
306 | |||
307 | pub fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) { | ||
308 | self.inner.leaf(kind, text) | ||
309 | } | ||
310 | |||
311 | pub fn start_branch(&mut self, kind: SyntaxKind) { | ||
312 | self.inner.start_internal(kind) | ||
313 | } | ||
314 | |||
315 | pub fn finish_branch(&mut self) { | ||
316 | self.inner.finish_internal() | ||
317 | } | ||
318 | |||
319 | pub fn error(&mut self, error: ParseError, text_pos: TextUnit) { | ||
320 | let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), text_pos); | ||
321 | self.errors.push(error) | ||
322 | } | ||
323 | } | ||