aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_syntax/src/parsing/reparsing.rs
diff options
context:
space:
mode:
Diffstat (limited to 'crates/ra_syntax/src/parsing/reparsing.rs')
-rw-r--r--crates/ra_syntax/src/parsing/reparsing.rs142
1 files changed, 80 insertions, 62 deletions
diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs
index 7e7f914f5..69887f500 100644
--- a/crates/ra_syntax/src/parsing/reparsing.rs
+++ b/crates/ra_syntax/src/parsing/reparsing.rs
@@ -12,7 +12,7 @@ use ra_parser::Reparser;
12use crate::{ 12use crate::{
13 SyntaxKind::*, TextRange, TextUnit, SyntaxError, 13 SyntaxKind::*, TextRange, TextUnit, SyntaxError,
14 algo, 14 algo,
15 syntax_node::{GreenNode, SyntaxNode}, 15 syntax_node::{GreenNode, SyntaxNode, GreenToken, SyntaxElement},
16 parsing::{ 16 parsing::{
17 text_token_source::TextTokenSource, 17 text_token_source::TextTokenSource,
18 text_tree_sink::TextTreeSink, 18 text_tree_sink::TextTreeSink,
@@ -24,60 +24,62 @@ pub(crate) fn incremental_reparse(
24 node: &SyntaxNode, 24 node: &SyntaxNode,
25 edit: &AtomTextEdit, 25 edit: &AtomTextEdit,
26 errors: Vec<SyntaxError>, 26 errors: Vec<SyntaxError>,
27) -> Option<(GreenNode, Vec<SyntaxError>)> { 27) -> Option<(GreenNode, Vec<SyntaxError>, TextRange)> {
28 let (node, green, new_errors) = 28 if let Some((green, old_range)) = reparse_token(node, &edit) {
29 reparse_leaf(node, &edit).or_else(|| reparse_block(node, &edit))?; 29 return Some((green, merge_errors(errors, Vec::new(), old_range, edit), old_range));
30 let green_root = node.replace_with(green); 30 }
31 let errors = merge_errors(errors, new_errors, node, edit); 31
32 Some((green_root, errors)) 32 if let Some((green, new_errors, old_range)) = reparse_block(node, &edit) {
33 return Some((green, merge_errors(errors, new_errors, old_range, edit), old_range));
34 }
35 None
33} 36}
34 37
35fn reparse_leaf<'node>( 38fn reparse_token<'node>(
36 root: &'node SyntaxNode, 39 root: &'node SyntaxNode,
37 edit: &AtomTextEdit, 40 edit: &AtomTextEdit,
38) -> Option<(&'node SyntaxNode, GreenNode, Vec<SyntaxError>)> { 41) -> Option<(GreenNode, TextRange)> {
39 let node = algo::find_covering_node(root, edit.delete); 42 let token = algo::find_covering_element(root, edit.delete).as_token()?;
40 match node.kind() { 43 match token.kind() {
41 WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => { 44 WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => {
42 if node.kind() == WHITESPACE || node.kind() == COMMENT { 45 if token.kind() == WHITESPACE || token.kind() == COMMENT {
43 // removing a new line may extends previous token 46 // removing a new line may extends previous token
44 if node.text().to_string()[edit.delete - node.range().start()].contains('\n') { 47 if token.text().to_string()[edit.delete - token.range().start()].contains('\n') {
45 return None; 48 return None;
46 } 49 }
47 } 50 }
48 51
49 let text = get_text_after_edit(node, &edit); 52 let text = get_text_after_edit(token.into(), &edit);
50 let tokens = tokenize(&text); 53 let lex_tokens = tokenize(&text);
51 let token = match tokens[..] { 54 let lex_token = match lex_tokens[..] {
52 [token] if token.kind == node.kind() => token, 55 [lex_token] if lex_token.kind == token.kind() => lex_token,
53 _ => return None, 56 _ => return None,
54 }; 57 };
55 58
56 if token.kind == IDENT && is_contextual_kw(&text) { 59 if lex_token.kind == IDENT && is_contextual_kw(&text) {
57 return None; 60 return None;
58 } 61 }
59 62
60 if let Some(next_char) = root.text().char_at(node.range().end()) { 63 if let Some(next_char) = root.text().char_at(token.range().end()) {
61 let tokens_with_next_char = tokenize(&format!("{}{}", text, next_char)); 64 let tokens_with_next_char = tokenize(&format!("{}{}", text, next_char));
62 if tokens_with_next_char.len() == 1 { 65 if tokens_with_next_char.len() == 1 {
63 return None; 66 return None;
64 } 67 }
65 } 68 }
66 69
67 let green = GreenNode::new_leaf(node.kind(), text.into()); 70 let new_token = GreenToken::new(token.kind(), text.into());
68 let new_errors = vec![]; 71 Some((token.replace_with(new_token), token.range()))
69 Some((node, green, new_errors))
70 } 72 }
71 _ => None, 73 _ => None,
72 } 74 }
73} 75}
74 76
75fn reparse_block<'node>( 77fn reparse_block<'node>(
76 node: &'node SyntaxNode, 78 root: &'node SyntaxNode,
77 edit: &AtomTextEdit, 79 edit: &AtomTextEdit,
78) -> Option<(&'node SyntaxNode, GreenNode, Vec<SyntaxError>)> { 80) -> Option<(GreenNode, Vec<SyntaxError>, TextRange)> {
79 let (node, reparser) = find_reparsable_node(node, edit.delete)?; 81 let (node, reparser) = find_reparsable_node(root, edit.delete)?;
80 let text = get_text_after_edit(node, &edit); 82 let text = get_text_after_edit(node.into(), &edit);
81 let tokens = tokenize(&text); 83 let tokens = tokenize(&text);
82 if !is_balanced(&tokens) { 84 if !is_balanced(&tokens) {
83 return None; 85 return None;
@@ -86,12 +88,16 @@ fn reparse_block<'node>(
86 let mut tree_sink = TextTreeSink::new(&text, &tokens); 88 let mut tree_sink = TextTreeSink::new(&text, &tokens);
87 reparser.parse(&token_source, &mut tree_sink); 89 reparser.parse(&token_source, &mut tree_sink);
88 let (green, new_errors) = tree_sink.finish(); 90 let (green, new_errors) = tree_sink.finish();
89 Some((node, green, new_errors)) 91 Some((node.replace_with(green), new_errors, node.range()))
90} 92}
91 93
92fn get_text_after_edit(node: &SyntaxNode, edit: &AtomTextEdit) -> String { 94fn get_text_after_edit(element: SyntaxElement, edit: &AtomTextEdit) -> String {
93 let edit = AtomTextEdit::replace(edit.delete - node.range().start(), edit.insert.clone()); 95 let edit = AtomTextEdit::replace(edit.delete - element.range().start(), edit.insert.clone());
94 edit.apply(node.text().to_string()) 96 let text = match element {
97 SyntaxElement::Token(token) => token.text().to_string(),
98 SyntaxElement::Node(node) => node.text().to_string(),
99 };
100 edit.apply(text)
95} 101}
96 102
97fn is_contextual_kw(text: &str) -> bool { 103fn is_contextual_kw(text: &str) -> bool {
@@ -102,9 +108,13 @@ fn is_contextual_kw(text: &str) -> bool {
102} 108}
103 109
104fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, Reparser)> { 110fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, Reparser)> {
105 let node = algo::find_covering_node(node, range); 111 let node = algo::find_covering_element(node, range);
106 node.ancestors().find_map(|node| { 112 let mut ancestors = match node {
107 let first_child = node.first_child().map(|it| it.kind()); 113 SyntaxElement::Token(it) => it.parent().ancestors(),
114 SyntaxElement::Node(it) => it.ancestors(),
115 };
116 ancestors.find_map(|node| {
117 let first_child = node.first_child_or_token().map(|it| it.kind());
108 let parent = node.parent().map(|it| it.kind()); 118 let parent = node.parent().map(|it| it.kind());
109 Reparser::for_node(node.kind(), first_child, parent).map(|r| (node, r)) 119 Reparser::for_node(node.kind(), first_child, parent).map(|r| (node, r))
110 }) 120 })
@@ -136,19 +146,19 @@ fn is_balanced(tokens: &[Token]) -> bool {
136fn merge_errors( 146fn merge_errors(
137 old_errors: Vec<SyntaxError>, 147 old_errors: Vec<SyntaxError>,
138 new_errors: Vec<SyntaxError>, 148 new_errors: Vec<SyntaxError>,
139 old_node: &SyntaxNode, 149 old_range: TextRange,
140 edit: &AtomTextEdit, 150 edit: &AtomTextEdit,
141) -> Vec<SyntaxError> { 151) -> Vec<SyntaxError> {
142 let mut res = Vec::new(); 152 let mut res = Vec::new();
143 for e in old_errors { 153 for e in old_errors {
144 if e.offset() <= old_node.range().start() { 154 if e.offset() <= old_range.start() {
145 res.push(e) 155 res.push(e)
146 } else if e.offset() >= old_node.range().end() { 156 } else if e.offset() >= old_range.end() {
147 res.push(e.add_offset(TextUnit::of_str(&edit.insert), edit.delete.len())); 157 res.push(e.add_offset(TextUnit::of_str(&edit.insert), edit.delete.len()));
148 } 158 }
149 } 159 }
150 for e in new_errors { 160 for e in new_errors {
151 res.push(e.add_offset(old_node.range().start(), 0.into())); 161 res.push(e.add_offset(old_range.start(), 0.into()));
152 } 162 }
153 res 163 res
154} 164}
@@ -160,13 +170,7 @@ mod tests {
160 use crate::{SourceFile, AstNode}; 170 use crate::{SourceFile, AstNode};
161 use super::*; 171 use super::*;
162 172
163 fn do_check<F>(before: &str, replace_with: &str, reparser: F) 173 fn do_check(before: &str, replace_with: &str, reparsed_len: u32) {
164 where
165 for<'a> F: Fn(
166 &'a SyntaxNode,
167 &AtomTextEdit,
168 ) -> Option<(&'a SyntaxNode, GreenNode, Vec<SyntaxError>)>,
169 {
170 let (range, before) = extract_range(before); 174 let (range, before) = extract_range(before);
171 let edit = AtomTextEdit::replace(range, replace_with.to_owned()); 175 let edit = AtomTextEdit::replace(range, replace_with.to_owned());
172 let after = edit.apply(before.clone()); 176 let after = edit.apply(before.clone());
@@ -175,23 +179,20 @@ mod tests {
175 let incrementally_reparsed = { 179 let incrementally_reparsed = {
176 let f = SourceFile::parse(&before); 180 let f = SourceFile::parse(&before);
177 let edit = AtomTextEdit { delete: range, insert: replace_with.to_string() }; 181 let edit = AtomTextEdit { delete: range, insert: replace_with.to_string() };
178 let (node, green, new_errors) = 182 let (green, new_errors, range) =
179 reparser(f.syntax(), &edit).expect("cannot incrementally reparse"); 183 incremental_reparse(f.syntax(), &edit, f.errors()).unwrap();
180 let green_root = node.replace_with(green); 184 assert_eq!(range.len(), reparsed_len.into(), "reparsed fragment has wrong length");
181 let errors = super::merge_errors(f.errors(), new_errors, node, &edit); 185 SourceFile::new(green, new_errors)
182 SourceFile::new(green_root, errors)
183 }; 186 };
184 187
185 assert_eq_text!( 188 assert_eq_text!(
186 &fully_reparsed.syntax().debug_dump(), 189 &fully_reparsed.syntax().debug_dump(),
187 &incrementally_reparsed.syntax().debug_dump(), 190 &incrementally_reparsed.syntax().debug_dump(),
188 ) 191 );
189 } 192 }
190 193
191 #[test] 194 #[test] // FIXME: some test here actually test token reparsing
192 fn reparse_block_tests() { 195 fn reparse_block_tests() {
193 let do_check = |before, replace_to| do_check(before, replace_to, reparse_block);
194
195 do_check( 196 do_check(
196 r" 197 r"
197fn foo() { 198fn foo() {
@@ -199,6 +200,7 @@ fn foo() {
199} 200}
200", 201",
201 "baz", 202 "baz",
203 3,
202 ); 204 );
203 do_check( 205 do_check(
204 r" 206 r"
@@ -207,6 +209,7 @@ fn foo() {
207} 209}
208", 210",
209 "baz", 211 "baz",
212 25,
210 ); 213 );
211 do_check( 214 do_check(
212 r" 215 r"
@@ -215,6 +218,7 @@ struct Foo {
215} 218}
216", 219",
217 ",\n g: (),", 220 ",\n g: (),",
221 14,
218 ); 222 );
219 do_check( 223 do_check(
220 r" 224 r"
@@ -225,6 +229,7 @@ fn foo {
225} 229}
226", 230",
227 "62", 231 "62",
232 31, // FIXME: reparse only int literal here
228 ); 233 );
229 do_check( 234 do_check(
230 r" 235 r"
@@ -233,7 +238,9 @@ mod foo {
233} 238}
234", 239",
235 "bar", 240 "bar",
241 11,
236 ); 242 );
243
237 do_check( 244 do_check(
238 r" 245 r"
239trait Foo { 246trait Foo {
@@ -241,6 +248,7 @@ trait Foo {
241} 248}
242", 249",
243 "Output", 250 "Output",
251 3,
244 ); 252 );
245 do_check( 253 do_check(
246 r" 254 r"
@@ -249,13 +257,9 @@ impl IntoIterator<Item=i32> for Foo {
249} 257}
250", 258",
251 "n next(", 259 "n next(",
260 9,
252 ); 261 );
253 do_check( 262 do_check(r"use a::b::{foo,<|>,bar<|>};", "baz", 10);
254 r"
255use a::b::{foo,<|>,bar<|>};
256 ",
257 "baz",
258 );
259 do_check( 263 do_check(
260 r" 264 r"
261pub enum A { 265pub enum A {
@@ -263,12 +267,14 @@ pub enum A {
263} 267}
264", 268",
265 "\nBar;\n", 269 "\nBar;\n",
270 11,
266 ); 271 );
267 do_check( 272 do_check(
268 r" 273 r"
269foo!{a, b<|><|> d} 274foo!{a, b<|><|> d}
270", 275",
271 ", c[3]", 276 ", c[3]",
277 8,
272 ); 278 );
273 do_check( 279 do_check(
274 r" 280 r"
@@ -277,6 +283,7 @@ fn foo() {
277} 283}
278", 284",
279 "123", 285 "123",
286 14,
280 ); 287 );
281 do_check( 288 do_check(
282 r" 289 r"
@@ -285,54 +292,60 @@ extern {
285} 292}
286", 293",
287 " exit(code: c_int)", 294 " exit(code: c_int)",
295 11,
288 ); 296 );
289 } 297 }
290 298
291 #[test] 299 #[test]
292 fn reparse_leaf_tests() { 300 fn reparse_token_tests() {
293 let do_check = |before, replace_to| do_check(before, replace_to, reparse_leaf);
294
295 do_check( 301 do_check(
296 r"<|><|> 302 r"<|><|>
297fn foo() -> i32 { 1 } 303fn foo() -> i32 { 1 }
298", 304",
299 "\n\n\n \n", 305 "\n\n\n \n",
306 1,
300 ); 307 );
301 do_check( 308 do_check(
302 r" 309 r"
303fn foo() -> <|><|> {} 310fn foo() -> <|><|> {}
304", 311",
305 " \n", 312 " \n",
313 2,
306 ); 314 );
307 do_check( 315 do_check(
308 r" 316 r"
309fn <|>foo<|>() -> i32 { 1 } 317fn <|>foo<|>() -> i32 { 1 }
310", 318",
311 "bar", 319 "bar",
320 3,
312 ); 321 );
313 do_check( 322 do_check(
314 r" 323 r"
315fn foo<|><|>foo() { } 324fn foo<|><|>foo() { }
316", 325",
317 "bar", 326 "bar",
327 6,
318 ); 328 );
319 do_check( 329 do_check(
320 r" 330 r"
321fn foo /* <|><|> */ () {} 331fn foo /* <|><|> */ () {}
322", 332",
323 "some comment", 333 "some comment",
334 6,
324 ); 335 );
325 do_check( 336 do_check(
326 r" 337 r"
327fn baz <|><|> () {} 338fn baz <|><|> () {}
328", 339",
329 " \t\t\n\n", 340 " \t\t\n\n",
341 2,
330 ); 342 );
331 do_check( 343 do_check(
332 r" 344 r"
333fn baz <|><|> () {} 345fn baz <|><|> () {}
334", 346",
335 " \t\t\n\n", 347 " \t\t\n\n",
348 2,
336 ); 349 );
337 do_check( 350 do_check(
338 r" 351 r"
@@ -340,24 +353,28 @@ fn baz <|><|> () {}
340mod { } 353mod { }
341", 354",
342 "c", 355 "c",
356 14,
343 ); 357 );
344 do_check( 358 do_check(
345 r#" 359 r#"
346fn -> &str { "Hello<|><|>" } 360fn -> &str { "Hello<|><|>" }
347"#, 361"#,
348 ", world", 362 ", world",
363 7,
349 ); 364 );
350 do_check( 365 do_check(
351 r#" 366 r#"
352fn -> &str { // "Hello<|><|>" 367fn -> &str { // "Hello<|><|>"
353"#, 368"#,
354 ", world", 369 ", world",
370 10,
355 ); 371 );
356 do_check( 372 do_check(
357 r##" 373 r##"
358fn -> &str { r#"Hello<|><|>"# 374fn -> &str { r#"Hello<|><|>"#
359"##, 375"##,
360 ", world", 376 ", world",
377 10,
361 ); 378 );
362 do_check( 379 do_check(
363 r" 380 r"
@@ -367,6 +384,7 @@ enum Foo {
367} 384}
368", 385",
369 "Clone", 386 "Clone",
387 4,
370 ); 388 );
371 } 389 }
372} 390}