aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crates/libsyntax2/src/lib.rs132
-rw-r--r--crates/libsyntax2/src/reparsing.rs343
-rw-r--r--crates/libsyntax2/src/text_utils.rs7
-rw-r--r--crates/libsyntax2/tests/test/main.rs76
4 files changed, 360 insertions, 198 deletions
diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs
index e761fa358..eb271762e 100644
--- a/crates/libsyntax2/src/lib.rs
+++ b/crates/libsyntax2/src/lib.rs
@@ -27,6 +27,10 @@ extern crate parking_lot;
27extern crate smol_str; 27extern crate smol_str;
28extern crate text_unit; 28extern crate text_unit;
29 29
30#[cfg(test)]
31#[macro_use]
32extern crate test_utils;
33
30pub mod algo; 34pub mod algo;
31pub mod ast; 35pub mod ast;
32mod lexer; 36mod lexer;
@@ -35,6 +39,7 @@ mod token_set;
35mod parser_api; 39mod parser_api;
36mod grammar; 40mod grammar;
37mod parser_impl; 41mod parser_impl;
42mod reparsing;
38 43
39mod syntax_kinds; 44mod syntax_kinds;
40mod yellow; 45mod yellow;
@@ -49,12 +54,11 @@ pub use {
49 lexer::{tokenize, Token}, 54 lexer::{tokenize, Token},
50 syntax_kinds::SyntaxKind, 55 syntax_kinds::SyntaxKind,
51 yellow::{SyntaxNode, SyntaxNodeRef, OwnedRoot, RefRoot, TreeRoot, SyntaxError}, 56 yellow::{SyntaxNode, SyntaxNodeRef, OwnedRoot, RefRoot, TreeRoot, SyntaxError},
57 reparsing::AtomEdit,
52}; 58};
53 59
54use { 60use {
55 SyntaxKind::*,
56 yellow::{GreenNode, SyntaxRoot}, 61 yellow::{GreenNode, SyntaxRoot},
57 parser_api::Parser,
58}; 62};
59 63
60#[derive(Clone, Debug, Hash)] 64#[derive(Clone, Debug, Hash)]
@@ -82,25 +86,11 @@ impl File {
82 self.incremental_reparse(edit).unwrap_or_else(|| self.full_reparse(edit)) 86 self.incremental_reparse(edit).unwrap_or_else(|| self.full_reparse(edit))
83 } 87 }
84 pub fn incremental_reparse(&self, edit: &AtomEdit) -> Option<File> { 88 pub fn incremental_reparse(&self, edit: &AtomEdit) -> Option<File> {
85 let (node, reparser) = find_reparsable_node(self.syntax(), edit.delete)?; 89 reparsing::incremental_reparse(self.syntax(), edit, self.errors())
86 let text = replace_range( 90 .map(|(green_node, errors)| File::new(green_node, errors))
87 node.text().to_string(),
88 edit.delete - node.range().start(),
89 &edit.insert,
90 );
91 let tokens = tokenize(&text);
92 if !is_balanced(&tokens) {
93 return None;
94 }
95 let (green, new_errors) = parser_impl::parse_with::<yellow::GreenBuilder>(
96 &text, &tokens, reparser,
97 );
98 let green_root = node.replace_with(green);
99 let errors = merge_errors(self.errors(), new_errors, node, edit);
100 Some(File::new(green_root, errors))
101 } 91 }
102 fn full_reparse(&self, edit: &AtomEdit) -> File { 92 fn full_reparse(&self, edit: &AtomEdit) -> File {
103 let text = replace_range(self.syntax().text().to_string(), edit.delete, &edit.insert); 93 let text = text_utils::replace_range(self.syntax().text().to_string(), edit.delete, &edit.insert);
104 File::parse(&text) 94 File::parse(&text)
105 } 95 }
106 pub fn ast(&self) -> ast::Root { 96 pub fn ast(&self) -> ast::Root {
@@ -113,107 +103,3 @@ impl File {
113 self.syntax().root.syntax_root().errors.clone() 103 self.syntax().root.syntax_root().errors.clone()
114 } 104 }
115} 105}
116
117#[derive(Debug, Clone)]
118pub struct AtomEdit {
119 pub delete: TextRange,
120 pub insert: String,
121}
122
123impl AtomEdit {
124 pub fn replace(range: TextRange, replace_with: String) -> AtomEdit {
125 AtomEdit { delete: range, insert: replace_with }
126 }
127
128 pub fn delete(range: TextRange) -> AtomEdit {
129 AtomEdit::replace(range, String::new())
130 }
131
132 pub fn insert(offset: TextUnit, text: String) -> AtomEdit {
133 AtomEdit::replace(TextRange::offset_len(offset, 0.into()), text)
134 }
135}
136
137fn find_reparsable_node(node: SyntaxNodeRef, range: TextRange) -> Option<(SyntaxNodeRef, fn(&mut Parser))> {
138 let node = algo::find_covering_node(node, range);
139 return algo::ancestors(node)
140 .filter_map(|node| reparser(node).map(|r| (node, r)))
141 .next();
142
143 fn reparser(node: SyntaxNodeRef) -> Option<fn(&mut Parser)> {
144 let res = match node.kind() {
145 BLOCK => grammar::block,
146 NAMED_FIELD_DEF_LIST => grammar::named_field_def_list,
147 NAMED_FIELD_LIST => grammar::named_field_list,
148 ENUM_VARIANT_LIST => grammar::enum_variant_list,
149 MATCH_ARM_LIST => grammar::match_arm_list,
150 USE_TREE_LIST => grammar::use_tree_list,
151 EXTERN_ITEM_LIST => grammar::extern_item_list,
152 TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree,
153 ITEM_LIST => {
154 let parent = node.parent().unwrap();
155 match parent.kind() {
156 IMPL_ITEM => grammar::impl_item_list,
157 TRAIT_DEF => grammar::trait_item_list,
158 MODULE => grammar::mod_item_list,
159 _ => return None,
160 }
161 },
162 _ => return None,
163 };
164 Some(res)
165 }
166}
167
168pub /*(meh)*/ fn replace_range(mut text: String, range: TextRange, replace_with: &str) -> String {
169 let start = u32::from(range.start()) as usize;
170 let end = u32::from(range.end()) as usize;
171 text.replace_range(start..end, replace_with);
172 text
173}
174
175fn is_balanced(tokens: &[Token]) -> bool {
176 if tokens.len() == 0
177 || tokens.first().unwrap().kind != L_CURLY
178 || tokens.last().unwrap().kind != R_CURLY {
179 return false
180 }
181 let mut balance = 0usize;
182 for t in tokens.iter() {
183 match t.kind {
184 L_CURLY => balance += 1,
185 R_CURLY => balance = match balance.checked_sub(1) {
186 Some(b) => b,
187 None => return false,
188 },
189 _ => (),
190 }
191 }
192 balance == 0
193}
194
195fn merge_errors(
196 old_errors: Vec<SyntaxError>,
197 new_errors: Vec<SyntaxError>,
198 old_node: SyntaxNodeRef,
199 edit: &AtomEdit,
200) -> Vec<SyntaxError> {
201 let mut res = Vec::new();
202 for e in old_errors {
203 if e.offset < old_node.range().start() {
204 res.push(e)
205 } else if e.offset > old_node.range().end() {
206 res.push(SyntaxError {
207 msg: e.msg,
208 offset: e.offset + TextUnit::of_str(&edit.insert) - edit.delete.len(),
209 })
210 }
211 }
212 for e in new_errors {
213 res.push(SyntaxError {
214 msg: e.msg,
215 offset: e.offset + old_node.range().start(),
216 })
217 }
218 res
219}
diff --git a/crates/libsyntax2/src/reparsing.rs b/crates/libsyntax2/src/reparsing.rs
new file mode 100644
index 000000000..da44913c5
--- /dev/null
+++ b/crates/libsyntax2/src/reparsing.rs
@@ -0,0 +1,343 @@
1use algo;
2use grammar;
3use lexer::{tokenize, Token};
4use text_unit::{TextRange, TextUnit};
5use yellow::{self, SyntaxNodeRef, GreenNode, SyntaxError};
6use parser_impl;
7use parser_api::Parser;
8use {
9 SyntaxKind::*,
10};
11use text_utils::replace_range;
12
13#[derive(Debug, Clone)]
14pub struct AtomEdit {
15 pub delete: TextRange,
16 pub insert: String,
17}
18
19impl AtomEdit {
20 pub fn replace(range: TextRange, replace_with: String) -> AtomEdit {
21 AtomEdit { delete: range, insert: replace_with }
22 }
23
24 pub fn delete(range: TextRange) -> AtomEdit {
25 AtomEdit::replace(range, String::new())
26 }
27
28 pub fn insert(offset: TextUnit, text: String) -> AtomEdit {
29 AtomEdit::replace(TextRange::offset_len(offset, 0.into()), text)
30 }
31}
32
33pub(crate) fn incremental_reparse(
34 node: SyntaxNodeRef,
35 edit: &AtomEdit,
36 errors: Vec<SyntaxError>,
37) -> Option<(GreenNode, Vec<SyntaxError>)> {
38 let (node, green, new_errors) =
39 reparse_leaf(node, &edit).or_else(|| reparse_block(node, &edit))?;
40 let green_root = node.replace_with(green);
41 let errors = merge_errors(errors, new_errors, node, edit);
42 Some((green_root, errors))
43}
44
45fn reparse_leaf<'node>(
46 node: SyntaxNodeRef<'node>,
47 edit: &AtomEdit,
48) -> Option<(SyntaxNodeRef<'node>, GreenNode, Vec<SyntaxError>)> {
49 let node = algo::find_covering_node(node, edit.delete);
50 match node.kind() {
51 | WHITESPACE
52 | COMMENT
53 | DOC_COMMENT
54 | IDENT
55 | STRING
56 | RAW_STRING => {
57 let text = get_text_after_edit(node, &edit);
58 let tokens = tokenize(&text);
59 let token = match tokens[..] {
60 [token] if token.kind == node.kind() => token,
61 _ => return None,
62 };
63
64 if token.kind == IDENT && is_contextual_kw(&text) {
65 return None;
66 }
67
68 let green = GreenNode::new_leaf(node.kind(), &text);
69 let new_errors = vec![];
70 Some((node, green, new_errors))
71 }
72 _ => None,
73 }
74}
75
76fn reparse_block<'node>(
77 node: SyntaxNodeRef<'node>,
78 edit: &AtomEdit,
79) -> Option<(SyntaxNodeRef<'node>, GreenNode, Vec<SyntaxError>)> {
80 let (node, reparser) = find_reparsable_node(node, edit.delete)?;
81 let text = get_text_after_edit(node, &edit);
82 let tokens = tokenize(&text);
83 if !is_balanced(&tokens) {
84 return None;
85 }
86 let (green, new_errors) =
87 parser_impl::parse_with::<yellow::GreenBuilder>(
88 &text, &tokens, reparser,
89 );
90 Some((node, green, new_errors))
91}
92
93fn get_text_after_edit(node: SyntaxNodeRef, edit: &AtomEdit) -> String {
94 replace_range(
95 node.text().to_string(),
96 edit.delete - node.range().start(),
97 &edit.insert,
98 )
99}
100
101fn is_contextual_kw(text: &str) -> bool {
102 match text {
103 | "auto"
104 | "default"
105 | "union" => true,
106 _ => false,
107 }
108}
109
110fn find_reparsable_node<'node>(
111 node: SyntaxNodeRef<'node>,
112 range: TextRange,
113) -> Option<(SyntaxNodeRef<'node>, fn(&mut Parser))> {
114 let node = algo::find_covering_node(node, range);
115 return algo::ancestors(node)
116 .filter_map(|node| reparser(node).map(|r| (node, r)))
117 .next();
118
119 fn reparser(node: SyntaxNodeRef) -> Option<fn(&mut Parser)> {
120 let res = match node.kind() {
121 BLOCK => grammar::block,
122 NAMED_FIELD_DEF_LIST => grammar::named_field_def_list,
123 NAMED_FIELD_LIST => grammar::named_field_list,
124 ENUM_VARIANT_LIST => grammar::enum_variant_list,
125 MATCH_ARM_LIST => grammar::match_arm_list,
126 USE_TREE_LIST => grammar::use_tree_list,
127 EXTERN_ITEM_LIST => grammar::extern_item_list,
128 TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree,
129 ITEM_LIST => {
130 let parent = node.parent().unwrap();
131 match parent.kind() {
132 IMPL_ITEM => grammar::impl_item_list,
133 TRAIT_DEF => grammar::trait_item_list,
134 MODULE => grammar::mod_item_list,
135 _ => return None,
136 }
137 }
138 _ => return None,
139 };
140 Some(res)
141 }
142}
143
144fn is_balanced(tokens: &[Token]) -> bool {
145 if tokens.len() == 0
146 || tokens.first().unwrap().kind != L_CURLY
147 || tokens.last().unwrap().kind != R_CURLY {
148 return false;
149 }
150 let mut balance = 0usize;
151 for t in tokens.iter() {
152 match t.kind {
153 L_CURLY => balance += 1,
154 R_CURLY => balance = match balance.checked_sub(1) {
155 Some(b) => b,
156 None => return false,
157 },
158 _ => (),
159 }
160 }
161 balance == 0
162}
163
164fn merge_errors(
165 old_errors: Vec<SyntaxError>,
166 new_errors: Vec<SyntaxError>,
167 old_node: SyntaxNodeRef,
168 edit: &AtomEdit,
169) -> Vec<SyntaxError> {
170 let mut res = Vec::new();
171 for e in old_errors {
172 if e.offset <= old_node.range().start() {
173 res.push(e)
174 } else if e.offset >= old_node.range().end() {
175 res.push(SyntaxError {
176 msg: e.msg,
177 offset: e.offset + TextUnit::of_str(&edit.insert) - edit.delete.len(),
178 })
179 }
180 }
181 for e in new_errors {
182 res.push(SyntaxError {
183 msg: e.msg,
184 offset: e.offset + old_node.range().start(),
185 })
186 }
187 res
188}
189
190#[cfg(test)]
191mod tests {
192 use super::{
193 super::{
194 File,
195 test_utils::extract_range,
196 text_utils::replace_range,
197 utils::dump_tree,
198 },
199 reparse_leaf, reparse_block, AtomEdit, GreenNode, SyntaxError, SyntaxNodeRef,
200 };
201
202 fn do_check<F>(
203 before: &str,
204 replace_with: &str,
205 reparser: F,
206 ) where
207 for<'a> F: Fn(
208 SyntaxNodeRef<'a>,
209 &AtomEdit,
210 ) -> Option<(SyntaxNodeRef<'a>, GreenNode, Vec<SyntaxError>)>
211 {
212 let (range, before) = extract_range(before);
213 let after = replace_range(before.clone(), range, replace_with);
214
215 let fully_reparsed = File::parse(&after);
216 let incrementally_reparsed = {
217 let f = File::parse(&before);
218 let edit = AtomEdit { delete: range, insert: replace_with.to_string() };
219 let (node, green, new_errors) =
220 reparser(f.syntax(), &edit).expect("cannot incrementally reparse");
221 let green_root = node.replace_with(green);
222 let errors = super::merge_errors(f.errors(), new_errors, node, &edit);
223 File::new(green_root, errors)
224 };
225
226 assert_eq_text!(
227 &dump_tree(fully_reparsed.syntax()),
228 &dump_tree(incrementally_reparsed.syntax()),
229 )
230 }
231
232 #[test]
233 fn reparse_block_tests() {
234 let do_check = |before, replace_to|
235 do_check(before, replace_to, reparse_block);
236
237 do_check(r"
238fn foo() {
239 let x = foo + <|>bar<|>
240}
241", "baz");
242 do_check(r"
243fn foo() {
244 let x = foo<|> + bar<|>
245}
246", "baz");
247 do_check(r"
248struct Foo {
249 f: foo<|><|>
250}
251", ",\n g: (),");
252 do_check(r"
253fn foo {
254 let;
255 1 + 1;
256 <|>92<|>;
257}
258", "62");
259 do_check(r"
260mod foo {
261 fn <|><|>
262}
263", "bar");
264 do_check(r"
265trait Foo {
266 type <|>Foo<|>;
267}
268", "Output");
269 do_check(r"
270impl IntoIterator<Item=i32> for Foo {
271 f<|><|>
272}
273", "n next(");
274 do_check(r"
275use a::b::{foo,<|>,bar<|>};
276 ", "baz");
277 do_check(r"
278pub enum A {
279 Foo<|><|>
280}
281", "\nBar;\n");
282 do_check(r"
283foo!{a, b<|><|> d}
284", ", c[3]");
285 do_check(r"
286fn foo() {
287 vec![<|><|>]
288}
289", "123");
290 do_check(r"
291extern {
292 fn<|>;<|>
293}
294", " exit(code: c_int)");
295 }
296
297 #[test]
298 fn reparse_leaf_tests() {
299 let do_check = |before, replace_to|
300 do_check(before, replace_to, reparse_leaf);
301
302 do_check(r"<|><|>
303fn foo() -> i32 { 1 }
304", "\n\n\n \n");
305 do_check(r"
306fn foo() -> <|><|> {}
307", " \n");
308 do_check(r"
309fn <|>foo<|>() -> i32 { 1 }
310", "bar");
311 do_check(r"
312fn foo<|><|>foo() { }
313", "bar");
314 do_check(r"
315fn foo /* <|><|> */ () {}
316", "some comment");
317 do_check(r"
318fn baz <|><|> () {}
319", " \t\t\n\n");
320 do_check(r"
321fn baz <|><|> () {}
322", " \t\t\n\n");
323 do_check(r"
324/// foo <|><|>omment
325mod { }
326", "c");
327 do_check(r#"
328fn -> &str { "Hello<|><|>" }
329"#, ", world");
330 do_check(r#"
331fn -> &str { // "Hello<|><|>"
332"#, ", world");
333 do_check(r##"
334fn -> &str { r#"Hello<|><|>"#
335"##, ", world");
336 do_check(r"
337#[derive(<|>Copy<|>)]
338enum Foo {
339
340}
341", "Clone");
342 }
343} \ No newline at end of file
diff --git a/crates/libsyntax2/src/text_utils.rs b/crates/libsyntax2/src/text_utils.rs
index e3d73888f..58ae1e43e 100644
--- a/crates/libsyntax2/src/text_utils.rs
+++ b/crates/libsyntax2/src/text_utils.rs
@@ -17,3 +17,10 @@ pub fn intersect(r1: TextRange, r2: TextRange) -> Option<TextRange> {
17 None 17 None
18 } 18 }
19} 19}
20
21pub fn replace_range(mut text: String, range: TextRange, replace_with: &str) -> String {
22 let start = u32::from(range.start()) as usize;
23 let end = u32::from(range.end()) as usize;
24 text.replace_range(start..end, replace_with);
25 text
26} \ No newline at end of file
diff --git a/crates/libsyntax2/tests/test/main.rs b/crates/libsyntax2/tests/test/main.rs
index 644df9f3c..5a8879fce 100644
--- a/crates/libsyntax2/tests/test/main.rs
+++ b/crates/libsyntax2/tests/test/main.rs
@@ -9,9 +9,8 @@ use std::{
9 fmt::Write, 9 fmt::Write,
10}; 10};
11 11
12use test_utils::extract_range;
13use libsyntax2::{ 12use libsyntax2::{
14 File, AtomEdit, 13 File,
15 utils::{dump_tree, check_fuzz_invariants}, 14 utils::{dump_tree, check_fuzz_invariants},
16}; 15};
17 16
@@ -24,79 +23,6 @@ fn lexer_tests() {
24} 23}
25 24
26#[test] 25#[test]
27fn reparse_test() {
28 fn do_check(before: &str, replace_with: &str) {
29 let (range, before) = extract_range(before);
30 let after = libsyntax2::replace_range(before.clone(), range, replace_with);
31
32 let fully_reparsed = File::parse(&after);
33 let incrementally_reparsed = {
34 let f = File::parse(&before);
35 let edit = AtomEdit { delete: range, insert: replace_with.to_string() };
36 f.incremental_reparse(&edit).unwrap()
37 };
38 assert_eq_text!(
39 &dump_tree(fully_reparsed.syntax()),
40 &dump_tree(incrementally_reparsed.syntax()),
41 )
42 }
43
44 do_check(r"
45fn foo() {
46 let x = foo + <|>bar<|>
47}
48", "baz");
49 do_check(r"
50struct Foo {
51 f: foo<|><|>
52}
53", ",\n g: (),");
54 do_check(r"
55fn foo {
56 let;
57 1 + 1;
58 <|>92<|>;
59}
60", "62");
61 do_check(r"
62mod foo {
63 fn <|><|>
64}
65", "bar");
66 do_check(r"
67trait Foo {
68 type <|>Foo<|>;
69}
70", "Output");
71 do_check(r"
72impl IntoIterator<Item=i32> for Foo {
73 f<|><|>
74}
75", "n next(");
76 do_check(r"
77use a::b::{foo,<|>,bar<|>};
78 ", "baz");
79 do_check(r"
80pub enum A {
81 Foo<|><|>
82}
83", "\nBar;\n");
84 do_check(r"
85foo!{a, b<|><|> d}
86", ", c[3]");
87 do_check(r"
88fn foo() {
89 vec![<|><|>]
90}
91", "123");
92 do_check(r"
93extern {
94 fn<|>;<|>
95}
96", " exit(code: c_int)");
97}
98
99#[test]
100fn parser_tests() { 26fn parser_tests() {
101 dir_tests(&["parser/inline", "parser/ok", "parser/err"], |text| { 27 dir_tests(&["parser/inline", "parser/ok", "parser/err"], |text| {
102 let file = File::parse(text); 28 let file = File::parse(text);