From 5222b8aba3b1c2c68706aacf6869423a8e4fe6d5 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 15:47:32 +0300 Subject: move all parsing related bits to a separate module --- crates/ra_syntax/src/parsing/reparsing.rs | 370 ++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 crates/ra_syntax/src/parsing/reparsing.rs (limited to 'crates/ra_syntax/src/parsing/reparsing.rs') diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs new file mode 100644 index 000000000..994e7e212 --- /dev/null +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -0,0 +1,370 @@ +use crate::{ + SyntaxKind::*, TextRange, TextUnit, + algo, + syntax_node::{GreenNode, SyntaxError, SyntaxNode}, + parsing::{ + grammar, + parser_impl, + builder::GreenBuilder, + parser_api::Parser, + lexer::{tokenize, Token}, + } +}; + +use ra_text_edit::AtomTextEdit; + +pub(crate) fn incremental_reparse( + node: &SyntaxNode, + edit: &AtomTextEdit, + errors: Vec, +) -> Option<(GreenNode, Vec)> { + let (node, green, new_errors) = + reparse_leaf(node, &edit).or_else(|| reparse_block(node, &edit))?; + let green_root = node.replace_with(green); + let errors = merge_errors(errors, new_errors, node, edit); + Some((green_root, errors)) +} + +fn reparse_leaf<'node>( + node: &'node SyntaxNode, + edit: &AtomTextEdit, +) -> Option<(&'node SyntaxNode, GreenNode, Vec)> { + let node = algo::find_covering_node(node, edit.delete); + match node.kind() { + WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => { + let text = get_text_after_edit(node, &edit); + let tokens = tokenize(&text); + let token = match tokens[..] { + [token] if token.kind == node.kind() => token, + _ => return None, + }; + + if token.kind == IDENT && is_contextual_kw(&text) { + return None; + } + + let green = GreenNode::new_leaf(node.kind(), text.into()); + let new_errors = vec![]; + Some((node, green, new_errors)) + } + _ => None, + } +} + +fn reparse_block<'node>( + node: &'node SyntaxNode, + edit: &AtomTextEdit, +) -> Option<(&'node SyntaxNode, GreenNode, Vec)> { + let (node, reparser) = find_reparsable_node(node, edit.delete)?; + let text = get_text_after_edit(node, &edit); + let tokens = tokenize(&text); + if !is_balanced(&tokens) { + return None; + } + let (green, new_errors) = + parser_impl::parse_with(GreenBuilder::new(), &text, &tokens, reparser); + Some((node, green, new_errors)) +} + +fn get_text_after_edit(node: &SyntaxNode, edit: &AtomTextEdit) -> String { + let edit = AtomTextEdit::replace(edit.delete - node.range().start(), edit.insert.clone()); + edit.apply(node.text().to_string()) +} + +fn is_contextual_kw(text: &str) -> bool { + match text { + "auto" | "default" | "union" => true, + _ => false, + } +} + +type ParseFn = fn(&mut Parser); +fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, ParseFn)> { + let node = algo::find_covering_node(node, range); + return node.ancestors().filter_map(|node| reparser(node).map(|r| (node, r))).next(); + + fn reparser(node: &SyntaxNode) -> Option { + let res = match node.kind() { + BLOCK => grammar::block, + NAMED_FIELD_DEF_LIST => grammar::named_field_def_list, + NAMED_FIELD_LIST => grammar::named_field_list, + ENUM_VARIANT_LIST => grammar::enum_variant_list, + MATCH_ARM_LIST => grammar::match_arm_list, + USE_TREE_LIST => grammar::use_tree_list, + EXTERN_ITEM_LIST => grammar::extern_item_list, + TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree, + ITEM_LIST => { + let parent = node.parent().unwrap(); + match parent.kind() { + IMPL_BLOCK => grammar::impl_item_list, + TRAIT_DEF => grammar::trait_item_list, + MODULE => grammar::mod_item_list, + _ => return None, + } + } + _ => return None, + }; + Some(res) + } +} + +fn is_balanced(tokens: &[Token]) -> bool { + if tokens.is_empty() + || tokens.first().unwrap().kind != L_CURLY + || tokens.last().unwrap().kind != R_CURLY + { + return false; + } + let mut balance = 0usize; + for t in tokens.iter() { + match t.kind { + L_CURLY => balance += 1, + R_CURLY => { + balance = match balance.checked_sub(1) { + Some(b) => b, + None => return false, + } + } + _ => (), + } + } + balance == 0 +} + +fn merge_errors( + old_errors: Vec, + new_errors: Vec, + old_node: &SyntaxNode, + edit: &AtomTextEdit, +) -> Vec { + let mut res = Vec::new(); + for e in old_errors { + if e.offset() <= old_node.range().start() { + res.push(e) + } else if e.offset() >= old_node.range().end() { + res.push(e.add_offset(TextUnit::of_str(&edit.insert) - edit.delete.len())); + } + } + for e in new_errors { + res.push(e.add_offset(old_node.range().start())); + } + res +} + +#[cfg(test)] +mod tests { + use test_utils::{extract_range, assert_eq_text}; + + use crate::{SourceFile, AstNode, utils::dump_tree}; + use super::*; + + fn do_check(before: &str, replace_with: &str, reparser: F) + where + for<'a> F: Fn( + &'a SyntaxNode, + &AtomTextEdit, + ) -> Option<(&'a SyntaxNode, GreenNode, Vec)>, + { + let (range, before) = extract_range(before); + let edit = AtomTextEdit::replace(range, replace_with.to_owned()); + let after = edit.apply(before.clone()); + + let fully_reparsed = SourceFile::parse(&after); + let incrementally_reparsed = { + let f = SourceFile::parse(&before); + let edit = AtomTextEdit { delete: range, insert: replace_with.to_string() }; + let (node, green, new_errors) = + reparser(f.syntax(), &edit).expect("cannot incrementally reparse"); + let green_root = node.replace_with(green); + let errors = super::merge_errors(f.errors(), new_errors, node, &edit); + SourceFile::new(green_root, errors) + }; + + assert_eq_text!( + &dump_tree(fully_reparsed.syntax()), + &dump_tree(incrementally_reparsed.syntax()), + ) + } + + #[test] + fn reparse_block_tests() { + let do_check = |before, replace_to| do_check(before, replace_to, reparse_block); + + do_check( + r" +fn foo() { + let x = foo + <|>bar<|> +} +", + "baz", + ); + do_check( + r" +fn foo() { + let x = foo<|> + bar<|> +} +", + "baz", + ); + do_check( + r" +struct Foo { + f: foo<|><|> +} +", + ",\n g: (),", + ); + do_check( + r" +fn foo { + let; + 1 + 1; + <|>92<|>; +} +", + "62", + ); + do_check( + r" +mod foo { + fn <|><|> +} +", + "bar", + ); + do_check( + r" +trait Foo { + type <|>Foo<|>; +} +", + "Output", + ); + do_check( + r" +impl IntoIterator for Foo { + f<|><|> +} +", + "n next(", + ); + do_check( + r" +use a::b::{foo,<|>,bar<|>}; + ", + "baz", + ); + do_check( + r" +pub enum A { + Foo<|><|> +} +", + "\nBar;\n", + ); + do_check( + r" +foo!{a, b<|><|> d} +", + ", c[3]", + ); + do_check( + r" +fn foo() { + vec![<|><|>] +} +", + "123", + ); + do_check( + r" +extern { + fn<|>;<|> +} +", + " exit(code: c_int)", + ); + } + + #[test] + fn reparse_leaf_tests() { + let do_check = |before, replace_to| do_check(before, replace_to, reparse_leaf); + + do_check( + r"<|><|> +fn foo() -> i32 { 1 } +", + "\n\n\n \n", + ); + do_check( + r" +fn foo() -> <|><|> {} +", + " \n", + ); + do_check( + r" +fn <|>foo<|>() -> i32 { 1 } +", + "bar", + ); + do_check( + r" +fn foo<|><|>foo() { } +", + "bar", + ); + do_check( + r" +fn foo /* <|><|> */ () {} +", + "some comment", + ); + do_check( + r" +fn baz <|><|> () {} +", + " \t\t\n\n", + ); + do_check( + r" +fn baz <|><|> () {} +", + " \t\t\n\n", + ); + do_check( + r" +/// foo <|><|>omment +mod { } +", + "c", + ); + do_check( + r#" +fn -> &str { "Hello<|><|>" } +"#, + ", world", + ); + do_check( + r#" +fn -> &str { // "Hello<|><|>" +"#, + ", world", + ); + do_check( + r##" +fn -> &str { r#"Hello<|><|>"# +"##, + ", world", + ); + do_check( + r" +#[derive(<|>Copy<|>)] +enum Foo { + +} +", + "Clone", + ); + } +} -- cgit v1.2.3 From 9f6883fbf10567e4177dbe77857c970b6dc61f48 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 16:02:06 +0300 Subject: move reparsers to grammar --- crates/ra_syntax/src/parsing/reparsing.rs | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) (limited to 'crates/ra_syntax/src/parsing/reparsing.rs') diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 994e7e212..0a24dae0e 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -81,31 +81,7 @@ fn is_contextual_kw(text: &str) -> bool { type ParseFn = fn(&mut Parser); fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, ParseFn)> { let node = algo::find_covering_node(node, range); - return node.ancestors().filter_map(|node| reparser(node).map(|r| (node, r))).next(); - - fn reparser(node: &SyntaxNode) -> Option { - let res = match node.kind() { - BLOCK => grammar::block, - NAMED_FIELD_DEF_LIST => grammar::named_field_def_list, - NAMED_FIELD_LIST => grammar::named_field_list, - ENUM_VARIANT_LIST => grammar::enum_variant_list, - MATCH_ARM_LIST => grammar::match_arm_list, - USE_TREE_LIST => grammar::use_tree_list, - EXTERN_ITEM_LIST => grammar::extern_item_list, - TOKEN_TREE if node.first_child().unwrap().kind() == L_CURLY => grammar::token_tree, - ITEM_LIST => { - let parent = node.parent().unwrap(); - match parent.kind() { - IMPL_BLOCK => grammar::impl_item_list, - TRAIT_DEF => grammar::trait_item_list, - MODULE => grammar::mod_item_list, - _ => return None, - } - } - _ => return None, - }; - Some(res) - } + node.ancestors().find_map(grammar::reparser).map(|r| (node, r)) } fn is_balanced(tokens: &[Token]) -> bool { -- cgit v1.2.3 From a4a1e08ab81193112a2e14413d084916241c3fca Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 16:16:14 +0300 Subject: flatten modules --- crates/ra_syntax/src/parsing/reparsing.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'crates/ra_syntax/src/parsing/reparsing.rs') diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 0a24dae0e..a88f53dae 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -1,7 +1,8 @@ use crate::{ SyntaxKind::*, TextRange, TextUnit, algo, - syntax_node::{GreenNode, SyntaxError, SyntaxNode}, + syntax_node::{GreenNode, SyntaxNode}, + syntax_error::SyntaxError, parsing::{ grammar, parser_impl, -- cgit v1.2.3 From 86a67dce25f11ba9803a5727f77c02fd1f49e2c0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 20 Feb 2019 16:48:06 +0300 Subject: fix tests --- crates/ra_syntax/src/parsing/reparsing.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'crates/ra_syntax/src/parsing/reparsing.rs') diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index a88f53dae..edf3fa291 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -79,10 +79,12 @@ fn is_contextual_kw(text: &str) -> bool { } } -type ParseFn = fn(&mut Parser); -fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, ParseFn)> { +fn find_reparsable_node( + node: &SyntaxNode, + range: TextRange, +) -> Option<(&SyntaxNode, fn(&mut Parser))> { let node = algo::find_covering_node(node, range); - node.ancestors().find_map(grammar::reparser).map(|r| (node, r)) + node.ancestors().find_map(|node| grammar::reparser(node).map(|r| (node, r))) } fn is_balanced(tokens: &[Token]) -> bool { -- cgit v1.2.3