diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-24 10:15:43 +0000 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-24 10:15:43 +0000 |
commit | f6f160391db945a0dcc2f73b38926d6919f7c566 (patch) | |
tree | 060450b70c09357615f261d8acd032a647615dd7 /crates/ra_syntax/src/parsing/text_tree_sink.rs | |
parent | c5e74cebdcbade069c0e1e81e298ab7d729e4cd5 (diff) | |
parent | 81bca78349afb9e15994f46401da0cfabfba04a1 (diff) |
Merge #885
885: Parse token trees directy r=matklad a=matklad
This takes advantage of the recent macro refactoring to directly parse token stream into a syntax tree.
Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_syntax/src/parsing/text_tree_sink.rs')
-rw-r--r-- | crates/ra_syntax/src/parsing/text_tree_sink.rs | 166 |
1 files changed, 166 insertions, 0 deletions
diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs new file mode 100644 index 000000000..961a91d41 --- /dev/null +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs | |||
@@ -0,0 +1,166 @@ | |||
1 | use std::mem; | ||
2 | |||
3 | use ra_parser::{TreeSink, ParseError}; | ||
4 | |||
5 | use crate::{ | ||
6 | SmolStr, SyntaxError, TextUnit, TextRange, SyntaxTreeBuilder, | ||
7 | SyntaxKind::{self, *}, | ||
8 | parsing::Token, | ||
9 | syntax_node::GreenNode, | ||
10 | }; | ||
11 | |||
12 | /// Bridges the parser with our specific syntax tree representation. | ||
13 | /// | ||
14 | /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. | ||
15 | pub(crate) struct TextTreeSink<'a> { | ||
16 | text: &'a str, | ||
17 | tokens: &'a [Token], | ||
18 | text_pos: TextUnit, | ||
19 | token_pos: usize, | ||
20 | state: State, | ||
21 | inner: SyntaxTreeBuilder, | ||
22 | } | ||
23 | |||
24 | enum State { | ||
25 | PendingStart, | ||
26 | Normal, | ||
27 | PendingFinish, | ||
28 | } | ||
29 | |||
30 | impl<'a> TreeSink for TextTreeSink<'a> { | ||
31 | fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { | ||
32 | match mem::replace(&mut self.state, State::Normal) { | ||
33 | State::PendingStart => unreachable!(), | ||
34 | State::PendingFinish => self.inner.finish_branch(), | ||
35 | State::Normal => (), | ||
36 | } | ||
37 | self.eat_trivias(); | ||
38 | let n_tokens = n_tokens as usize; | ||
39 | let len = self.tokens[self.token_pos..self.token_pos + n_tokens] | ||
40 | .iter() | ||
41 | .map(|it| it.len) | ||
42 | .sum::<TextUnit>(); | ||
43 | self.do_leaf(kind, len, n_tokens); | ||
44 | } | ||
45 | |||
46 | fn start_branch(&mut self, kind: SyntaxKind) { | ||
47 | match mem::replace(&mut self.state, State::Normal) { | ||
48 | State::PendingStart => { | ||
49 | self.inner.start_branch(kind); | ||
50 | // No need to attach trivias to previous node: there is no | ||
51 | // previous node. | ||
52 | return; | ||
53 | } | ||
54 | State::PendingFinish => self.inner.finish_branch(), | ||
55 | State::Normal => (), | ||
56 | } | ||
57 | |||
58 | let n_trivias = | ||
59 | self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); | ||
60 | let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; | ||
61 | let mut trivia_end = | ||
62 | self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>(); | ||
63 | |||
64 | let n_attached_trivias = { | ||
65 | let leading_trivias = leading_trivias.iter().rev().map(|it| { | ||
66 | let next_end = trivia_end - it.len; | ||
67 | let range = TextRange::from_to(next_end, trivia_end); | ||
68 | trivia_end = next_end; | ||
69 | (it.kind, &self.text[range]) | ||
70 | }); | ||
71 | n_attached_trivias(kind, leading_trivias) | ||
72 | }; | ||
73 | self.eat_n_trivias(n_trivias - n_attached_trivias); | ||
74 | self.inner.start_branch(kind); | ||
75 | self.eat_n_trivias(n_attached_trivias); | ||
76 | } | ||
77 | |||
78 | fn finish_branch(&mut self) { | ||
79 | match mem::replace(&mut self.state, State::PendingFinish) { | ||
80 | State::PendingStart => unreachable!(), | ||
81 | State::PendingFinish => self.inner.finish_branch(), | ||
82 | State::Normal => (), | ||
83 | } | ||
84 | } | ||
85 | |||
86 | fn error(&mut self, error: ParseError) { | ||
87 | self.inner.error(error, self.text_pos) | ||
88 | } | ||
89 | } | ||
90 | |||
91 | impl<'a> TextTreeSink<'a> { | ||
92 | pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> TextTreeSink<'a> { | ||
93 | TextTreeSink { | ||
94 | text, | ||
95 | tokens, | ||
96 | text_pos: 0.into(), | ||
97 | token_pos: 0, | ||
98 | state: State::PendingStart, | ||
99 | inner: SyntaxTreeBuilder::default(), | ||
100 | } | ||
101 | } | ||
102 | |||
103 | pub(super) fn finish(mut self) -> (GreenNode, Vec<SyntaxError>) { | ||
104 | match mem::replace(&mut self.state, State::Normal) { | ||
105 | State::PendingFinish => { | ||
106 | self.eat_trivias(); | ||
107 | self.inner.finish_branch() | ||
108 | } | ||
109 | State::PendingStart | State::Normal => unreachable!(), | ||
110 | } | ||
111 | |||
112 | self.inner.finish_raw() | ||
113 | } | ||
114 | |||
115 | fn eat_trivias(&mut self) { | ||
116 | while let Some(&token) = self.tokens.get(self.token_pos) { | ||
117 | if !token.kind.is_trivia() { | ||
118 | break; | ||
119 | } | ||
120 | self.do_leaf(token.kind, token.len, 1); | ||
121 | } | ||
122 | } | ||
123 | |||
124 | fn eat_n_trivias(&mut self, n: usize) { | ||
125 | for _ in 0..n { | ||
126 | let token = self.tokens[self.token_pos]; | ||
127 | assert!(token.kind.is_trivia()); | ||
128 | self.do_leaf(token.kind, token.len, 1); | ||
129 | } | ||
130 | } | ||
131 | |||
132 | fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { | ||
133 | let range = TextRange::offset_len(self.text_pos, len); | ||
134 | let text: SmolStr = self.text[range].into(); | ||
135 | self.text_pos += len; | ||
136 | self.token_pos += n_tokens; | ||
137 | self.inner.leaf(kind, text); | ||
138 | } | ||
139 | } | ||
140 | |||
141 | fn n_attached_trivias<'a>( | ||
142 | kind: SyntaxKind, | ||
143 | trivias: impl Iterator<Item = (SyntaxKind, &'a str)>, | ||
144 | ) -> usize { | ||
145 | match kind { | ||
146 | CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF | ||
147 | | MODULE | NAMED_FIELD_DEF => { | ||
148 | let mut res = 0; | ||
149 | for (i, (kind, text)) in trivias.enumerate() { | ||
150 | match kind { | ||
151 | WHITESPACE => { | ||
152 | if text.contains("\n\n") { | ||
153 | break; | ||
154 | } | ||
155 | } | ||
156 | COMMENT => { | ||
157 | res = i + 1; | ||
158 | } | ||
159 | _ => (), | ||
160 | } | ||
161 | } | ||
162 | res | ||
163 | } | ||
164 | _ => 0, | ||
165 | } | ||
166 | } | ||