diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-21 09:25:20 +0000 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-02-21 09:25:20 +0000 |
commit | 18b0c509f77a8e06141fee6668532cced1ebf5d8 (patch) | |
tree | e6b1718e4557a33921bf6637e76bf74c0d3cfa96 /crates/ra_syntax/src/parsing/event.rs | |
parent | c84561bb624280b84eb2fe6c6b2a6b9fe3f1dbf7 (diff) | |
parent | cd0d2866fc61e03aeca7bd7d1bc652e0443d2b3d (diff) |
Merge #865
865: Parser decoupling r=matklad a=matklad
More work on making parser independent from text
Co-authored-by: Aleksey Kladov <[email protected]>
Diffstat (limited to 'crates/ra_syntax/src/parsing/event.rs')
-rw-r--r-- | crates/ra_syntax/src/parsing/event.rs | 193 |
1 files changed, 37 insertions, 156 deletions
diff --git a/crates/ra_syntax/src/parsing/event.rs b/crates/ra_syntax/src/parsing/event.rs index f6f020eab..d6cbdffe0 100644 --- a/crates/ra_syntax/src/parsing/event.rs +++ b/crates/ra_syntax/src/parsing/event.rs | |||
@@ -10,13 +10,8 @@ | |||
10 | use std::mem; | 10 | use std::mem; |
11 | 11 | ||
12 | use crate::{ | 12 | use crate::{ |
13 | SmolStr, | ||
14 | SyntaxKind::{self, *}, | 13 | SyntaxKind::{self, *}, |
15 | TextRange, TextUnit, | 14 | parsing::{ParseError, TreeSink}, |
16 | parsing::{ | ||
17 | ParseError, TreeSink, | ||
18 | lexer::Token, | ||
19 | }, | ||
20 | }; | 15 | }; |
21 | 16 | ||
22 | /// `Parser` produces a flat list of `Event`s. | 17 | /// `Parser` produces a flat list of `Event`s. |
@@ -88,160 +83,46 @@ impl Event { | |||
88 | } | 83 | } |
89 | } | 84 | } |
90 | 85 | ||
91 | pub(super) struct EventProcessor<'a, S: TreeSink> { | 86 | /// Generate the syntax tree with the control of events. |
92 | sink: S, | 87 | pub(super) fn process(sink: &mut impl TreeSink, mut events: Vec<Event>) { |
93 | text_pos: TextUnit, | 88 | let mut forward_parents = Vec::new(); |
94 | text: &'a str, | 89 | |
95 | token_pos: usize, | 90 | for i in 0..events.len() { |
96 | tokens: &'a [Token], | 91 | match mem::replace(&mut events[i], Event::tombstone()) { |
97 | events: &'a mut [Event], | 92 | Event::Start { kind: TOMBSTONE, .. } => (), |
98 | } | 93 | |
99 | 94 | Event::Start { kind, forward_parent } => { | |
100 | impl<'a, S: TreeSink> EventProcessor<'a, S> { | 95 | // For events[A, B, C], B is A's forward_parent, C is B's forward_parent, |
101 | pub(super) fn new( | 96 | // in the normal control flow, the parent-child relation: `A -> B -> C`, |
102 | sink: S, | 97 | // while with the magic forward_parent, it writes: `C <- B <- A`. |
103 | text: &'a str, | 98 | |
104 | tokens: &'a [Token], | 99 | // append `A` into parents. |
105 | events: &'a mut [Event], | 100 | forward_parents.push(kind); |
106 | ) -> EventProcessor<'a, S> { | 101 | let mut idx = i; |
107 | EventProcessor { sink, text_pos: 0.into(), text, token_pos: 0, tokens, events } | 102 | let mut fp = forward_parent; |
108 | } | 103 | while let Some(fwd) = fp { |
109 | 104 | idx += fwd as usize; | |
110 | /// Generate the syntax tree with the control of events. | 105 | // append `A`'s forward_parent `B` |
111 | pub(crate) fn process(mut self) -> S { | 106 | fp = match mem::replace(&mut events[idx], Event::tombstone()) { |
112 | let mut forward_parents = Vec::new(); | 107 | Event::Start { kind, forward_parent } => { |
113 | 108 | forward_parents.push(kind); | |
114 | for i in 0..self.events.len() { | 109 | forward_parent |
115 | match mem::replace(&mut self.events[i], Event::tombstone()) { | 110 | } |
116 | Event::Start { kind: TOMBSTONE, .. } => (), | 111 | _ => unreachable!(), |
117 | 112 | }; | |
118 | Event::Start { kind, forward_parent } => { | 113 | // append `B`'s forward_parent `C` in the next stage. |
119 | // For events[A, B, C], B is A's forward_parent, C is B's forward_parent, | ||
120 | // in the normal control flow, the parent-child relation: `A -> B -> C`, | ||
121 | // while with the magic forward_parent, it writes: `C <- B <- A`. | ||
122 | |||
123 | // append `A` into parents. | ||
124 | forward_parents.push(kind); | ||
125 | let mut idx = i; | ||
126 | let mut fp = forward_parent; | ||
127 | while let Some(fwd) = fp { | ||
128 | idx += fwd as usize; | ||
129 | // append `A`'s forward_parent `B` | ||
130 | fp = match mem::replace(&mut self.events[idx], Event::tombstone()) { | ||
131 | Event::Start { kind, forward_parent } => { | ||
132 | forward_parents.push(kind); | ||
133 | forward_parent | ||
134 | } | ||
135 | _ => unreachable!(), | ||
136 | }; | ||
137 | // append `B`'s forward_parent `C` in the next stage. | ||
138 | } | ||
139 | |||
140 | for kind in forward_parents.drain(..).rev() { | ||
141 | self.start(kind); | ||
142 | } | ||
143 | } | ||
144 | Event::Finish => { | ||
145 | let is_last = i == self.events.len() - 1; | ||
146 | self.finish(is_last); | ||
147 | } | ||
148 | Event::Token { kind, n_raw_tokens } => { | ||
149 | self.eat_trivias(); | ||
150 | let n_raw_tokens = n_raw_tokens as usize; | ||
151 | let len = self.tokens[self.token_pos..self.token_pos + n_raw_tokens] | ||
152 | .iter() | ||
153 | .map(|it| it.len) | ||
154 | .sum::<TextUnit>(); | ||
155 | self.leaf(kind, len, n_raw_tokens); | ||
156 | } | 114 | } |
157 | Event::Error { msg } => self.sink.error(msg), | ||
158 | } | ||
159 | } | ||
160 | self.sink | ||
161 | } | ||
162 | |||
163 | /// Add the node into syntax tree but discard the comments/whitespaces. | ||
164 | fn start(&mut self, kind: SyntaxKind) { | ||
165 | if kind == SOURCE_FILE { | ||
166 | self.sink.start_branch(kind); | ||
167 | return; | ||
168 | } | ||
169 | let n_trivias = | ||
170 | self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); | ||
171 | let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; | ||
172 | let mut trivia_end = | ||
173 | self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextUnit>(); | ||
174 | 115 | ||
175 | let n_attached_trivias = { | 116 | for (j, kind) in forward_parents.drain(..).rev().enumerate() { |
176 | let leading_trivias = leading_trivias.iter().rev().map(|it| { | 117 | let is_root_node = i == 0 && j == 0; |
177 | let next_end = trivia_end - it.len; | 118 | sink.start_branch(kind, is_root_node); |
178 | let range = TextRange::from_to(next_end, trivia_end); | ||
179 | trivia_end = next_end; | ||
180 | (it.kind, &self.text[range]) | ||
181 | }); | ||
182 | n_attached_trivias(kind, leading_trivias) | ||
183 | }; | ||
184 | self.eat_n_trivias(n_trivias - n_attached_trivias); | ||
185 | self.sink.start_branch(kind); | ||
186 | self.eat_n_trivias(n_attached_trivias); | ||
187 | } | ||
188 | |||
189 | fn finish(&mut self, is_last: bool) { | ||
190 | if is_last { | ||
191 | self.eat_trivias() | ||
192 | } | ||
193 | self.sink.finish_branch(); | ||
194 | } | ||
195 | |||
196 | fn eat_trivias(&mut self) { | ||
197 | while let Some(&token) = self.tokens.get(self.token_pos) { | ||
198 | if !token.kind.is_trivia() { | ||
199 | break; | ||
200 | } | ||
201 | self.leaf(token.kind, token.len, 1); | ||
202 | } | ||
203 | } | ||
204 | |||
205 | fn eat_n_trivias(&mut self, n: usize) { | ||
206 | for _ in 0..n { | ||
207 | let token = self.tokens[self.token_pos]; | ||
208 | assert!(token.kind.is_trivia()); | ||
209 | self.leaf(token.kind, token.len, 1); | ||
210 | } | ||
211 | } | ||
212 | |||
213 | fn leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { | ||
214 | let range = TextRange::offset_len(self.text_pos, len); | ||
215 | let text: SmolStr = self.text[range].into(); | ||
216 | self.text_pos += len; | ||
217 | self.token_pos += n_tokens; | ||
218 | self.sink.leaf(kind, text); | ||
219 | } | ||
220 | } | ||
221 | |||
222 | fn n_attached_trivias<'a>( | ||
223 | kind: SyntaxKind, | ||
224 | trivias: impl Iterator<Item = (SyntaxKind, &'a str)>, | ||
225 | ) -> usize { | ||
226 | match kind { | ||
227 | CONST_DEF | TYPE_DEF | STRUCT_DEF | ENUM_DEF | ENUM_VARIANT | FN_DEF | TRAIT_DEF | ||
228 | | MODULE | NAMED_FIELD_DEF => { | ||
229 | let mut res = 0; | ||
230 | for (i, (kind, text)) in trivias.enumerate() { | ||
231 | match kind { | ||
232 | WHITESPACE => { | ||
233 | if text.contains("\n\n") { | ||
234 | break; | ||
235 | } | ||
236 | } | ||
237 | COMMENT => { | ||
238 | res = i + 1; | ||
239 | } | ||
240 | _ => (), | ||
241 | } | 119 | } |
242 | } | 120 | } |
243 | res | 121 | Event::Finish => sink.finish_branch(i == events.len() - 1), |
122 | Event::Token { kind, n_raw_tokens } => { | ||
123 | sink.leaf(kind, n_raw_tokens); | ||
124 | } | ||
125 | Event::Error { msg } => sink.error(msg), | ||
244 | } | 126 | } |
245 | _ => 0, | ||
246 | } | 127 | } |
247 | } | 128 | } |