From 4356240fa42eebbd17fdb63b2d0e6f86387ca21c Mon Sep 17 00:00:00 2001 From: darksv Date: Thu, 13 Sep 2018 23:25:05 +0200 Subject: Incremental reparsing for single tokens (WHITESPACE, COMMENT, DOC_COMMENT, IDENT, STRING, RAW_STRING) --- crates/libsyntax2/src/lib.rs | 80 ++++++++++++++++++++++++++++++------ crates/libsyntax2/tests/test/main.rs | 56 ++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 14 deletions(-) diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs index fd58cb4fa..bae685fb4 100644 --- a/crates/libsyntax2/src/lib.rs +++ b/crates/libsyntax2/src/lib.rs @@ -82,22 +82,68 @@ impl File { self.incremental_reparse(edit).unwrap_or_else(|| self.full_reparse(edit)) } pub fn incremental_reparse(&self, edit: &AtomEdit) -> Option { + let (node, green, new_errors) = + self.reparse_leaf(&edit).or_else(|| self.reparse_block(&edit))?; + + let green_root = node.replace_with(green); + let errors = merge_errors(self.errors(), new_errors, node, edit); + Some(File::new(green_root, errors)) + } + fn reparse_leaf(&self, edit: &AtomEdit) -> Option<(SyntaxNodeRef, GreenNode, Vec)> { + let node = algo::find_covering_node(self.syntax(), edit.delete); + match node.kind() { + | WHITESPACE + | COMMENT + | DOC_COMMENT + | IDENT + | STRING + | RAW_STRING => { + let text = get_text_after_edit(node, &edit); + let tokens = tokenize(&text); + if tokens.len() != 1 || tokens[0].kind != node.kind() { + return None; + } + + let reparser: fn(&mut Parser) = if node.kind().is_trivia() { + // since trivia is omitted by parser when it doesn't have a parent, \ + // we need to create one for it + |p| { + p.start().complete(p, ROOT); + } + } else { + |p| { + p.bump(); + } + }; + + let (green, new_errors) = + parser_impl::parse_with::( + &text, &tokens, reparser, + ); + + let green = if node.kind().is_trivia() { + green.children().first().cloned().unwrap() + } else { + green + }; + + Some((node, green, new_errors)) + }, + _ => None, + } + } + fn reparse_block(&self, edit: &AtomEdit) -> Option<(SyntaxNodeRef, GreenNode, Vec)> { let (node, reparser) = find_reparsable_node(self.syntax(), edit.delete)?; - let text = replace_range( - node.text().to_string(), - edit.delete - node.range().start(), - &edit.insert, - ); + let text = get_text_after_edit(node, &edit); let tokens = tokenize(&text); if !is_balanced(&tokens) { return None; } - let (green, new_errors) = parser_impl::parse_with::( - &text, &tokens, reparser, - ); - let green_root = node.replace_with(green); - let errors = merge_errors(self.errors(), new_errors, node, edit); - Some(File::new(green_root, errors)) + let (green, new_errors) = + parser_impl::parse_with::( + &text, &tokens, reparser, + ); + Some((node, green, new_errors)) } fn full_reparse(&self, edit: &AtomEdit) -> File { let text = replace_range(self.syntax().text().to_string(), edit.delete, &edit.insert); @@ -134,6 +180,14 @@ impl AtomEdit { } } +fn get_text_after_edit(node: SyntaxNodeRef, edit: &AtomEdit) -> String { + replace_range( + node.text().to_string(), + edit.delete - node.range().start(), + &edit.insert, + ) +} + fn find_reparsable_node(node: SyntaxNodeRef, range: TextRange) -> Option<(SyntaxNodeRef, fn(&mut Parser))> { let node = algo::find_covering_node(node, range); return algo::ancestors(node) @@ -200,9 +254,9 @@ fn merge_errors( ) -> Vec { let mut res = Vec::new(); for e in old_errors { - if e.offset < old_node.range().start() { + if e.offset <= old_node.range().start() { res.push(e) - } else if e.offset > old_node.range().end() { + } else if e.offset >= old_node.range().end() { res.push(SyntaxError { msg: e.msg, offset: e.offset + TextUnit::of_str(&edit.insert) - edit.delete.len(), diff --git a/crates/libsyntax2/tests/test/main.rs b/crates/libsyntax2/tests/test/main.rs index 644df9f3c..ce7e075f8 100644 --- a/crates/libsyntax2/tests/test/main.rs +++ b/crates/libsyntax2/tests/test/main.rs @@ -33,7 +33,7 @@ fn reparse_test() { let incrementally_reparsed = { let f = File::parse(&before); let edit = AtomEdit { delete: range, insert: replace_with.to_string() }; - f.incremental_reparse(&edit).unwrap() + f.incremental_reparse(&edit).expect("cannot incrementally reparse") }; assert_eq_text!( &dump_tree(fully_reparsed.syntax()), @@ -45,6 +45,11 @@ fn reparse_test() { fn foo() { let x = foo + <|>bar<|> } +", "baz"); + do_check(r" +fn foo() { + let x = foo<|> + bar<|> +} ", "baz"); do_check(r" struct Foo { @@ -67,6 +72,11 @@ mod foo { trait Foo { type <|>Foo<|>; } +", "Output"); + do_check(r" +trait Foo { + type<|> Foo<|>; +} ", "Output"); do_check(r" impl IntoIterator for Foo { @@ -94,6 +104,50 @@ extern { fn<|>;<|> } ", " exit(code: c_int)"); +do_check(r"<|><|> +fn foo() -> i32 { + 1 +} +", "\n\n\n \n"); + do_check(r" +fn foo() -> <|><|> {} +", " \n"); + do_check(r" +fn <|>foo<|>() -> i32 { + 1 +} +", "bar"); +do_check(r" +fn aa<|><|>bb() { + +} +", "foofoo"); + do_check(r" +fn aabb /* <|><|> */ () { + +} +", "some comment"); + do_check(r" +fn aabb <|><|> () { + +} +", " \t\t\n\n"); + do_check(r" +trait foo { +// comment <|><|> +} +", "\n"); + do_check(r" +/// good <|><|>omment +mod { +} +", "c"); + do_check(r#" +fn -> &str { "Hello<|><|>" } +"#, ", world"); + do_check(r#" +fn -> &str { // "Hello<|><|>" +"#, ", world"); } #[test] -- cgit v1.2.3 From c300135322730cd685b03d1adc8b8c7c7e714b0a Mon Sep 17 00:00:00 2001 From: darksv Date: Fri, 14 Sep 2018 18:09:33 +0200 Subject: create leaf directly without calling the parser --- crates/libsyntax2/src/lib.rs | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs index bae685fb4..d124d3284 100644 --- a/crates/libsyntax2/src/lib.rs +++ b/crates/libsyntax2/src/lib.rs @@ -104,29 +104,8 @@ impl File { return None; } - let reparser: fn(&mut Parser) = if node.kind().is_trivia() { - // since trivia is omitted by parser when it doesn't have a parent, \ - // we need to create one for it - |p| { - p.start().complete(p, ROOT); - } - } else { - |p| { - p.bump(); - } - }; - - let (green, new_errors) = - parser_impl::parse_with::( - &text, &tokens, reparser, - ); - - let green = if node.kind().is_trivia() { - green.children().first().cloned().unwrap() - } else { - green - }; - + let green = GreenNode::new_leaf(node.kind(), &text); + let new_errors = vec![]; Some((node, green, new_errors)) }, _ => None, -- cgit v1.2.3 From bc94bf95ce858ef247ebee006d50cfdc33f6bf5f Mon Sep 17 00:00:00 2001 From: darksv Date: Fri, 14 Sep 2018 19:26:48 +0200 Subject: correctly handle IDENTs when changed to contextual keywords --- crates/libsyntax2/src/lib.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs index d124d3284..7c9fbc421 100644 --- a/crates/libsyntax2/src/lib.rs +++ b/crates/libsyntax2/src/lib.rs @@ -100,7 +100,12 @@ impl File { | RAW_STRING => { let text = get_text_after_edit(node, &edit); let tokens = tokenize(&text); - if tokens.len() != 1 || tokens[0].kind != node.kind() { + let token = match tokens[..] { + [token] if token.kind == node.kind() => token, + _ => return None, + }; + + if token.kind == IDENT && is_contextual_kw(&text) { return None; } @@ -167,6 +172,15 @@ fn get_text_after_edit(node: SyntaxNodeRef, edit: &AtomEdit) -> String { ) } +fn is_contextual_kw(text: &str) -> bool { + match text { + | "auto" + | "default" + | "union" => true, + _ => false, + } +} + fn find_reparsable_node(node: SyntaxNodeRef, range: TextRange) -> Option<(SyntaxNodeRef, fn(&mut Parser))> { let node = algo::find_covering_node(node, range); return algo::ancestors(node) -- cgit v1.2.3 From a29211918b728fb83246bfcb43d2ad9c79e182fb Mon Sep 17 00:00:00 2001 From: darksv Date: Sat, 15 Sep 2018 13:35:55 +0200 Subject: create separated mod for reparsing functionality --- crates/libsyntax2/src/lib.rs | 175 +-------------------------------- crates/libsyntax2/src/reparsing.rs | 188 ++++++++++++++++++++++++++++++++++++ crates/libsyntax2/src/text_utils.rs | 7 ++ 3 files changed, 200 insertions(+), 170 deletions(-) create mode 100644 crates/libsyntax2/src/reparsing.rs diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs index 7c9fbc421..014cdafee 100644 --- a/crates/libsyntax2/src/lib.rs +++ b/crates/libsyntax2/src/lib.rs @@ -35,6 +35,7 @@ mod token_set; mod parser_api; mod grammar; mod parser_impl; +mod reparsing; mod syntax_kinds; mod yellow; @@ -49,12 +50,11 @@ pub use { lexer::{tokenize, Token}, syntax_kinds::SyntaxKind, yellow::{SyntaxNode, SyntaxNodeRef, OwnedRoot, RefRoot, TreeRoot, SyntaxError}, + reparsing::AtomEdit, }; use { - SyntaxKind::*, yellow::{GreenNode, SyntaxRoot}, - parser_api::Parser, }; #[derive(Clone, Debug)] @@ -82,55 +82,11 @@ impl File { self.incremental_reparse(edit).unwrap_or_else(|| self.full_reparse(edit)) } pub fn incremental_reparse(&self, edit: &AtomEdit) -> Option { - let (node, green, new_errors) = - self.reparse_leaf(&edit).or_else(|| self.reparse_block(&edit))?; - - let green_root = node.replace_with(green); - let errors = merge_errors(self.errors(), new_errors, node, edit); - Some(File::new(green_root, errors)) - } - fn reparse_leaf(&self, edit: &AtomEdit) -> Option<(SyntaxNodeRef, GreenNode, Vec)> { - let node = algo::find_covering_node(self.syntax(), edit.delete); - match node.kind() { - | WHITESPACE - | COMMENT - | DOC_COMMENT - | IDENT - | STRING - | RAW_STRING => { - let text = get_text_after_edit(node, &edit); - let tokens = tokenize(&text); - let token = match tokens[..] { - [token] if token.kind == node.kind() => token, - _ => return None, - }; - - if token.kind == IDENT && is_contextual_kw(&text) { - return None; - } - - let green = GreenNode::new_leaf(node.kind(), &text); - let new_errors = vec![]; - Some((node, green, new_errors)) - }, - _ => None, - } - } - fn reparse_block(&self, edit: &AtomEdit) -> Option<(SyntaxNodeRef, GreenNode, Vec)> { - let (node, reparser) = find_reparsable_node(self.syntax(), edit.delete)?; - let text = get_text_after_edit(node, &edit); - let tokens = tokenize(&text); - if !is_balanced(&tokens) { - return None; - } - let (green, new_errors) = - parser_impl::parse_with::( - &text, &tokens, reparser, - ); - Some((node, green, new_errors)) + reparsing::incremental_reparse(self.syntax(), edit, self.errors()) + .map(|(green_node, errors)| File::new(green_node, errors)) } fn full_reparse(&self, edit: &AtomEdit) -> File { - let text = replace_range(self.syntax().text().to_string(), edit.delete, &edit.insert); + let text = text_utils::replace_range(self.syntax().text().to_string(), edit.delete, &edit.insert); File::parse(&text) } pub fn ast(&self) -> ast::Root { @@ -143,124 +99,3 @@ impl File { self.syntax().root.syntax_root().errors.clone() } } - -#[derive(Debug, Clone)] -pub struct AtomEdit { - pub delete: TextRange, - pub insert: String, -} - -impl AtomEdit { - pub fn replace(range: TextRange, replace_with: String) -> AtomEdit { - AtomEdit { delete: range, insert: replace_with } - } - - pub fn delete(range: TextRange) -> AtomEdit { - AtomEdit::replace(range, String::new()) - } - - pub fn insert(offset: TextUnit, text: String) -> AtomEdit { - AtomEdit::replace(TextRange::offset_len(offset, 0.into()), text) - } -} - -fn get_text_after_edit(node: SyntaxNodeRef, edit: &AtomEdit) -> String { - replace_range( - node.text().to_string(), - edit.delete - node.range().start(), - &edit.insert, - ) -} - -fn is_contextual_kw(text: &str) -> bool { - match text { - | "auto" - | "default" - | "union" => true, - _ => false, - } -} - -fn find_reparsable_node(node: SyntaxNodeRef, range: TextRange) -> Option<(SyntaxNodeRef, fn(&mut Parser))> { - let node = algo::find_covering_node(node, range); - return algo::ancestors(node) - .filter_map(|node| reparser(node).map(|r| (node, r))) - .next(); - - fn reparser(node: SyntaxNodeRef) -> Option { - let res = match node.kind() { - BLOCK => grammar::block, - NAMED_FIELD_DEF_LIST => grammar::named_field_def_list, - NAMED_FIELD_LIST => grammar::named_field_list, - ENUM_VARIANT_LIST => grammar::enum_variant_list, - MATCH_ARM_LIST => grammar::match_arm_list, - USE_TREE_LIST => grammar::use_tree_list, - EXTERN_ITEM_LIST => grammar::extern_item_list, - TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree, - ITEM_LIST => { - let parent = node.parent().unwrap(); - match parent.kind() { - IMPL_ITEM => grammar::impl_item_list, - TRAIT_DEF => grammar::trait_item_list, - MODULE => grammar::mod_item_list, - _ => return None, - } - }, - _ => return None, - }; - Some(res) - } -} - -pub /*(meh)*/ fn replace_range(mut text: String, range: TextRange, replace_with: &str) -> String { - let start = u32::from(range.start()) as usize; - let end = u32::from(range.end()) as usize; - text.replace_range(start..end, replace_with); - text -} - -fn is_balanced(tokens: &[Token]) -> bool { - if tokens.len() == 0 - || tokens.first().unwrap().kind != L_CURLY - || tokens.last().unwrap().kind != R_CURLY { - return false - } - let mut balance = 0usize; - for t in tokens.iter() { - match t.kind { - L_CURLY => balance += 1, - R_CURLY => balance = match balance.checked_sub(1) { - Some(b) => b, - None => return false, - }, - _ => (), - } - } - balance == 0 -} - -fn merge_errors( - old_errors: Vec, - new_errors: Vec, - old_node: SyntaxNodeRef, - edit: &AtomEdit, -) -> Vec { - let mut res = Vec::new(); - for e in old_errors { - if e.offset <= old_node.range().start() { - res.push(e) - } else if e.offset >= old_node.range().end() { - res.push(SyntaxError { - msg: e.msg, - offset: e.offset + TextUnit::of_str(&edit.insert) - edit.delete.len(), - }) - } - } - for e in new_errors { - res.push(SyntaxError { - msg: e.msg, - offset: e.offset + old_node.range().start(), - }) - } - res -} diff --git a/crates/libsyntax2/src/reparsing.rs b/crates/libsyntax2/src/reparsing.rs new file mode 100644 index 000000000..723ea2b8b --- /dev/null +++ b/crates/libsyntax2/src/reparsing.rs @@ -0,0 +1,188 @@ +use algo; +use grammar; +use lexer::{tokenize, Token}; +use text_unit::{TextRange, TextUnit}; +use yellow::{self, SyntaxNodeRef, GreenNode, SyntaxError}; +use parser_impl; +use parser_api::Parser; +use { + SyntaxKind::*, +}; +use text_utils::replace_range; + +#[derive(Debug, Clone)] +pub struct AtomEdit { + pub delete: TextRange, + pub insert: String, +} + +impl AtomEdit { + pub fn replace(range: TextRange, replace_with: String) -> AtomEdit { + AtomEdit { delete: range, insert: replace_with } + } + + pub fn delete(range: TextRange) -> AtomEdit { + AtomEdit::replace(range, String::new()) + } + + pub fn insert(offset: TextUnit, text: String) -> AtomEdit { + AtomEdit::replace(TextRange::offset_len(offset, 0.into()), text) + } +} + +pub(crate) fn incremental_reparse( + node: SyntaxNodeRef, + edit: &AtomEdit, + errors: Vec, +) -> Option<(GreenNode, Vec)> { + let (node, green, new_errors) = + reparse_leaf(node, &edit).or_else(|| reparse_block(node, &edit))?; + let green_root = node.replace_with(green); + let errors = merge_errors(errors, new_errors, node, edit); + Some((green_root, errors)) +} + +fn reparse_leaf<'node>( + node: SyntaxNodeRef<'node>, + edit: &AtomEdit, +) -> Option<(SyntaxNodeRef<'node>, GreenNode, Vec)> { + let node = algo::find_covering_node(node, edit.delete); + match node.kind() { + | WHITESPACE + | COMMENT + | DOC_COMMENT + | IDENT + | STRING + | RAW_STRING => { + let text = get_text_after_edit(node, &edit); + let tokens = tokenize(&text); + let token = match tokens[..] { + [token] if token.kind == node.kind() => token, + _ => return None, + }; + + if token.kind == IDENT && is_contextual_kw(&text) { + return None; + } + + let green = GreenNode::new_leaf(node.kind(), &text); + let new_errors = vec![]; + Some((node, green, new_errors)) + } + _ => None, + } +} + +fn reparse_block<'node>( + node: SyntaxNodeRef<'node>, + edit: &AtomEdit, +) -> Option<(SyntaxNodeRef<'node>, GreenNode, Vec)> { + let (node, reparser) = find_reparsable_node(node, edit.delete)?; + let text = get_text_after_edit(node, &edit); + let tokens = tokenize(&text); + if !is_balanced(&tokens) { + return None; + } + let (green, new_errors) = + parser_impl::parse_with::( + &text, &tokens, reparser, + ); + Some((node, green, new_errors)) +} + +fn get_text_after_edit(node: SyntaxNodeRef, edit: &AtomEdit) -> String { + replace_range( + node.text().to_string(), + edit.delete - node.range().start(), + &edit.insert, + ) +} + +fn is_contextual_kw(text: &str) -> bool { + match text { + | "auto" + | "default" + | "union" => true, + _ => false, + } +} + +fn find_reparsable_node<'node>( + node: SyntaxNodeRef<'node>, + range: TextRange, +) -> Option<(SyntaxNodeRef<'node>, fn(&mut Parser))> { + let node = algo::find_covering_node(node, range); + return algo::ancestors(node) + .filter_map(|node| reparser(node).map(|r| (node, r))) + .next(); + + fn reparser(node: SyntaxNodeRef) -> Option { + let res = match node.kind() { + BLOCK => grammar::block, + NAMED_FIELD_DEF_LIST => grammar::named_field_def_list, + NAMED_FIELD_LIST => grammar::named_field_list, + ENUM_VARIANT_LIST => grammar::enum_variant_list, + MATCH_ARM_LIST => grammar::match_arm_list, + USE_TREE_LIST => grammar::use_tree_list, + EXTERN_ITEM_LIST => grammar::extern_item_list, + TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree, + ITEM_LIST => { + let parent = node.parent().unwrap(); + match parent.kind() { + IMPL_ITEM => grammar::impl_item_list, + TRAIT_DEF => grammar::trait_item_list, + MODULE => grammar::mod_item_list, + _ => return None, + } + } + _ => return None, + }; + Some(res) + } +} + +fn is_balanced(tokens: &[Token]) -> bool { + if tokens.len() == 0 + || tokens.first().unwrap().kind != L_CURLY + || tokens.last().unwrap().kind != R_CURLY { + return false; + } + let mut balance = 0usize; + for t in tokens.iter() { + match t.kind { + L_CURLY => balance += 1, + R_CURLY => balance = match balance.checked_sub(1) { + Some(b) => b, + None => return false, + }, + _ => (), + } + } + balance == 0 +} + +fn merge_errors( + old_errors: Vec, + new_errors: Vec, + old_node: SyntaxNodeRef, + edit: &AtomEdit, +) -> Vec { + let mut res = Vec::new(); + for e in old_errors { + if e.offset <= old_node.range().start() { + res.push(e) + } else if e.offset >= old_node.range().end() { + res.push(SyntaxError { + msg: e.msg, + offset: e.offset + TextUnit::of_str(&edit.insert) - edit.delete.len(), + }) + } + } + for e in new_errors { + res.push(SyntaxError { + msg: e.msg, + offset: e.offset + old_node.range().start(), + }) + } + res +} diff --git a/crates/libsyntax2/src/text_utils.rs b/crates/libsyntax2/src/text_utils.rs index e3d73888f..58ae1e43e 100644 --- a/crates/libsyntax2/src/text_utils.rs +++ b/crates/libsyntax2/src/text_utils.rs @@ -17,3 +17,10 @@ pub fn intersect(r1: TextRange, r2: TextRange) -> Option { None } } + +pub fn replace_range(mut text: String, range: TextRange, replace_with: &str) -> String { + let start = u32::from(range.start()) as usize; + let end = u32::from(range.end()) as usize; + text.replace_range(start..end, replace_with); + text +} \ No newline at end of file -- cgit v1.2.3 From 16ad5384f0c0427922b2d2d05025412fc8e4addf Mon Sep 17 00:00:00 2001 From: darksv Date: Sat, 15 Sep 2018 13:42:10 +0200 Subject: commit missing file --- crates/libsyntax2/tests/test/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/libsyntax2/tests/test/main.rs b/crates/libsyntax2/tests/test/main.rs index ce7e075f8..2b68673c6 100644 --- a/crates/libsyntax2/tests/test/main.rs +++ b/crates/libsyntax2/tests/test/main.rs @@ -27,7 +27,7 @@ fn lexer_tests() { fn reparse_test() { fn do_check(before: &str, replace_with: &str) { let (range, before) = extract_range(before); - let after = libsyntax2::replace_range(before.clone(), range, replace_with); + let after = libsyntax2::text_utils::replace_range(before.clone(), range, replace_with); let fully_reparsed = File::parse(&after); let incrementally_reparsed = { -- cgit v1.2.3 From 46cee0415c957f3262ad68cbe617b642f29f347e Mon Sep 17 00:00:00 2001 From: darksv Date: Sat, 15 Sep 2018 14:35:30 +0200 Subject: move reparsing tests --- crates/libsyntax2/src/lib.rs | 4 ++ crates/libsyntax2/src/reparsing.rs | 135 +++++++++++++++++++++++++++++++++++ crates/libsyntax2/tests/test/main.rs | 130 +-------------------------------- 3 files changed, 140 insertions(+), 129 deletions(-) diff --git a/crates/libsyntax2/src/lib.rs b/crates/libsyntax2/src/lib.rs index 014cdafee..886195660 100644 --- a/crates/libsyntax2/src/lib.rs +++ b/crates/libsyntax2/src/lib.rs @@ -27,6 +27,10 @@ extern crate parking_lot; extern crate smol_str; extern crate text_unit; +#[cfg(test)] +#[macro_use] +extern crate test_utils; + pub mod algo; pub mod ast; mod lexer; diff --git a/crates/libsyntax2/src/reparsing.rs b/crates/libsyntax2/src/reparsing.rs index 723ea2b8b..1df2ca094 100644 --- a/crates/libsyntax2/src/reparsing.rs +++ b/crates/libsyntax2/src/reparsing.rs @@ -186,3 +186,138 @@ fn merge_errors( } res } + +#[cfg(test)] +use super::{File, text_utils, test_utils, utils}; + +#[cfg(test)] +mod tests { + use super::{*, utils::dump_tree}; + + #[test] + fn reparse_test() { + fn do_check(before: &str, replace_with: &str) { + let (range, before) = test_utils::extract_range(before); + let after = text_utils::replace_range(before.clone(), range, replace_with); + + let fully_reparsed = File::parse(&after); + let incrementally_reparsed = { + let f = File::parse(&before); + let edit = AtomEdit { delete: range, insert: replace_with.to_string() }; + f.incremental_reparse(&edit).expect("cannot incrementally reparse") + }; + assert_eq_text!( + &dump_tree(fully_reparsed.syntax()), + &dump_tree(incrementally_reparsed.syntax()), + ) + } + + do_check(r" +fn foo() { + let x = foo + <|>bar<|> +} +", "baz"); + do_check(r" +fn foo() { + let x = foo<|> + bar<|> +} +", "baz"); + do_check(r" +struct Foo { + f: foo<|><|> +} +", ",\n g: (),"); + do_check(r" +fn foo { + let; + 1 + 1; + <|>92<|>; +} +", "62"); + do_check(r" +mod foo { + fn <|><|> +} +", "bar"); + do_check(r" +trait Foo { + type <|>Foo<|>; +} +", "Output"); + do_check(r" +trait Foo { + type<|> Foo<|>; +} +", "Output"); + do_check(r" +impl IntoIterator for Foo { + f<|><|> +} +", "n next("); + do_check(r" +use a::b::{foo,<|>,bar<|>}; + ", "baz"); + do_check(r" +pub enum A { + Foo<|><|> +} +", "\nBar;\n"); + do_check(r" +foo!{a, b<|><|> d} +", ", c[3]"); + do_check(r" +fn foo() { + vec![<|><|>] +} +", "123"); + do_check(r" +extern { + fn<|>;<|> +} +", " exit(code: c_int)"); + do_check(r"<|><|> +fn foo() -> i32 { + 1 +} +", "\n\n\n \n"); + do_check(r" +fn foo() -> <|><|> {} +", " \n"); + do_check(r" +fn <|>foo<|>() -> i32 { + 1 +} +", "bar"); + do_check(r" +fn aa<|><|>bb() { + +} +", "foofoo"); + do_check(r" +fn aabb /* <|><|> */ () { + +} +", "some comment"); + do_check(r" +fn aabb <|><|> () { + +} +", " \t\t\n\n"); + do_check(r" +trait foo { +// comment <|><|> +} +", "\n"); + do_check(r" +/// good <|><|>omment +mod { +} +", "c"); + do_check(r#" +fn -> &str { "Hello<|><|>" } +"#, ", world"); + do_check(r#" +fn -> &str { // "Hello<|><|>" +"#, ", world"); + } +} \ No newline at end of file diff --git a/crates/libsyntax2/tests/test/main.rs b/crates/libsyntax2/tests/test/main.rs index 2b68673c6..5a8879fce 100644 --- a/crates/libsyntax2/tests/test/main.rs +++ b/crates/libsyntax2/tests/test/main.rs @@ -9,9 +9,8 @@ use std::{ fmt::Write, }; -use test_utils::extract_range; use libsyntax2::{ - File, AtomEdit, + File, utils::{dump_tree, check_fuzz_invariants}, }; @@ -23,133 +22,6 @@ fn lexer_tests() { }) } -#[test] -fn reparse_test() { - fn do_check(before: &str, replace_with: &str) { - let (range, before) = extract_range(before); - let after = libsyntax2::text_utils::replace_range(before.clone(), range, replace_with); - - let fully_reparsed = File::parse(&after); - let incrementally_reparsed = { - let f = File::parse(&before); - let edit = AtomEdit { delete: range, insert: replace_with.to_string() }; - f.incremental_reparse(&edit).expect("cannot incrementally reparse") - }; - assert_eq_text!( - &dump_tree(fully_reparsed.syntax()), - &dump_tree(incrementally_reparsed.syntax()), - ) - } - - do_check(r" -fn foo() { - let x = foo + <|>bar<|> -} -", "baz"); - do_check(r" -fn foo() { - let x = foo<|> + bar<|> -} -", "baz"); - do_check(r" -struct Foo { - f: foo<|><|> -} -", ",\n g: (),"); - do_check(r" -fn foo { - let; - 1 + 1; - <|>92<|>; -} -", "62"); - do_check(r" -mod foo { - fn <|><|> -} -", "bar"); - do_check(r" -trait Foo { - type <|>Foo<|>; -} -", "Output"); - do_check(r" -trait Foo { - type<|> Foo<|>; -} -", "Output"); - do_check(r" -impl IntoIterator for Foo { - f<|><|> -} -", "n next("); - do_check(r" -use a::b::{foo,<|>,bar<|>}; - ", "baz"); - do_check(r" -pub enum A { - Foo<|><|> -} -", "\nBar;\n"); - do_check(r" -foo!{a, b<|><|> d} -", ", c[3]"); - do_check(r" -fn foo() { - vec![<|><|>] -} -", "123"); - do_check(r" -extern { - fn<|>;<|> -} -", " exit(code: c_int)"); -do_check(r"<|><|> -fn foo() -> i32 { - 1 -} -", "\n\n\n \n"); - do_check(r" -fn foo() -> <|><|> {} -", " \n"); - do_check(r" -fn <|>foo<|>() -> i32 { - 1 -} -", "bar"); -do_check(r" -fn aa<|><|>bb() { - -} -", "foofoo"); - do_check(r" -fn aabb /* <|><|> */ () { - -} -", "some comment"); - do_check(r" -fn aabb <|><|> () { - -} -", " \t\t\n\n"); - do_check(r" -trait foo { -// comment <|><|> -} -", "\n"); - do_check(r" -/// good <|><|>omment -mod { -} -", "c"); - do_check(r#" -fn -> &str { "Hello<|><|>" } -"#, ", world"); - do_check(r#" -fn -> &str { // "Hello<|><|>" -"#, ", world"); -} - #[test] fn parser_tests() { dir_tests(&["parser/inline", "parser/ok", "parser/err"], |text| { -- cgit v1.2.3 From ab00639032981f8b959e06c4015dd72201df651d Mon Sep 17 00:00:00 2001 From: darksv Date: Sat, 15 Sep 2018 17:05:08 +0200 Subject: independent tests for incremental reparsing of blocks and leaves --- crates/libsyntax2/src/reparsing.rs | 116 ++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 48 deletions(-) diff --git a/crates/libsyntax2/src/reparsing.rs b/crates/libsyntax2/src/reparsing.rs index 1df2ca094..da44913c5 100644 --- a/crates/libsyntax2/src/reparsing.rs +++ b/crates/libsyntax2/src/reparsing.rs @@ -187,30 +187,52 @@ fn merge_errors( res } -#[cfg(test)] -use super::{File, text_utils, test_utils, utils}; - #[cfg(test)] mod tests { - use super::{*, utils::dump_tree}; + use super::{ + super::{ + File, + test_utils::extract_range, + text_utils::replace_range, + utils::dump_tree, + }, + reparse_leaf, reparse_block, AtomEdit, GreenNode, SyntaxError, SyntaxNodeRef, + }; - #[test] - fn reparse_test() { - fn do_check(before: &str, replace_with: &str) { - let (range, before) = test_utils::extract_range(before); - let after = text_utils::replace_range(before.clone(), range, replace_with); + fn do_check( + before: &str, + replace_with: &str, + reparser: F, + ) where + for<'a> F: Fn( + SyntaxNodeRef<'a>, + &AtomEdit, + ) -> Option<(SyntaxNodeRef<'a>, GreenNode, Vec)> + { + let (range, before) = extract_range(before); + let after = replace_range(before.clone(), range, replace_with); - let fully_reparsed = File::parse(&after); - let incrementally_reparsed = { - let f = File::parse(&before); - let edit = AtomEdit { delete: range, insert: replace_with.to_string() }; - f.incremental_reparse(&edit).expect("cannot incrementally reparse") - }; - assert_eq_text!( - &dump_tree(fully_reparsed.syntax()), - &dump_tree(incrementally_reparsed.syntax()), - ) - } + let fully_reparsed = File::parse(&after); + let incrementally_reparsed = { + let f = File::parse(&before); + let edit = AtomEdit { delete: range, insert: replace_with.to_string() }; + let (node, green, new_errors) = + reparser(f.syntax(), &edit).expect("cannot incrementally reparse"); + let green_root = node.replace_with(green); + let errors = super::merge_errors(f.errors(), new_errors, node, &edit); + File::new(green_root, errors) + }; + + assert_eq_text!( + &dump_tree(fully_reparsed.syntax()), + &dump_tree(incrementally_reparsed.syntax()), + ) + } + + #[test] + fn reparse_block_tests() { + let do_check = |before, replace_to| + do_check(before, replace_to, reparse_block); do_check(r" fn foo() { @@ -243,11 +265,6 @@ mod foo { trait Foo { type <|>Foo<|>; } -", "Output"); - do_check(r" -trait Foo { - type<|> Foo<|>; -} ", "Output"); do_check(r" impl IntoIterator for Foo { @@ -275,43 +292,37 @@ extern { fn<|>;<|> } ", " exit(code: c_int)"); + } + + #[test] + fn reparse_leaf_tests() { + let do_check = |before, replace_to| + do_check(before, replace_to, reparse_leaf); + do_check(r"<|><|> -fn foo() -> i32 { - 1 -} +fn foo() -> i32 { 1 } ", "\n\n\n \n"); do_check(r" fn foo() -> <|><|> {} ", " \n"); do_check(r" -fn <|>foo<|>() -> i32 { - 1 -} +fn <|>foo<|>() -> i32 { 1 } ", "bar"); do_check(r" -fn aa<|><|>bb() { - -} -", "foofoo"); +fn foo<|><|>foo() { } +", "bar"); do_check(r" -fn aabb /* <|><|> */ () { - -} +fn foo /* <|><|> */ () {} ", "some comment"); do_check(r" -fn aabb <|><|> () { - -} +fn baz <|><|> () {} ", " \t\t\n\n"); do_check(r" -trait foo { -// comment <|><|> -} -", "\n"); +fn baz <|><|> () {} +", " \t\t\n\n"); do_check(r" -/// good <|><|>omment -mod { -} +/// foo <|><|>omment +mod { } ", "c"); do_check(r#" fn -> &str { "Hello<|><|>" } @@ -319,5 +330,14 @@ fn -> &str { "Hello<|><|>" } do_check(r#" fn -> &str { // "Hello<|><|>" "#, ", world"); + do_check(r##" +fn -> &str { r#"Hello<|><|>"# +"##, ", world"); + do_check(r" +#[derive(<|>Copy<|>)] +enum Foo { + +} +", "Clone"); } } \ No newline at end of file -- cgit v1.2.3