From 9e213385c9d06db3c8ca20812779e2b8f8ad2c71 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 30 Mar 2019 13:25:53 +0300 Subject: switch to new rowan --- crates/ra_syntax/src/parsing/reparsing.rs | 142 ++++++++++++++----------- crates/ra_syntax/src/parsing/text_tree_sink.rs | 28 ++--- 2 files changed, 94 insertions(+), 76 deletions(-) (limited to 'crates/ra_syntax/src/parsing') diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 7e7f914f5..69887f500 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -12,7 +12,7 @@ use ra_parser::Reparser; use crate::{ SyntaxKind::*, TextRange, TextUnit, SyntaxError, algo, - syntax_node::{GreenNode, SyntaxNode}, + syntax_node::{GreenNode, SyntaxNode, GreenToken, SyntaxElement}, parsing::{ text_token_source::TextTokenSource, text_tree_sink::TextTreeSink, @@ -24,60 +24,62 @@ pub(crate) fn incremental_reparse( node: &SyntaxNode, edit: &AtomTextEdit, errors: Vec, -) -> Option<(GreenNode, Vec)> { - let (node, green, new_errors) = - reparse_leaf(node, &edit).or_else(|| reparse_block(node, &edit))?; - let green_root = node.replace_with(green); - let errors = merge_errors(errors, new_errors, node, edit); - Some((green_root, errors)) +) -> Option<(GreenNode, Vec, TextRange)> { + if let Some((green, old_range)) = reparse_token(node, &edit) { + return Some((green, merge_errors(errors, Vec::new(), old_range, edit), old_range)); + } + + if let Some((green, new_errors, old_range)) = reparse_block(node, &edit) { + return Some((green, merge_errors(errors, new_errors, old_range, edit), old_range)); + } + None } -fn reparse_leaf<'node>( +fn reparse_token<'node>( root: &'node SyntaxNode, edit: &AtomTextEdit, -) -> Option<(&'node SyntaxNode, GreenNode, Vec)> { - let node = algo::find_covering_node(root, edit.delete); - match node.kind() { +) -> Option<(GreenNode, TextRange)> { + let token = algo::find_covering_element(root, edit.delete).as_token()?; + match token.kind() { WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => { - if node.kind() == WHITESPACE || node.kind() == COMMENT { + if token.kind() == WHITESPACE || token.kind() == COMMENT { // removing a new line may extends previous token - if node.text().to_string()[edit.delete - node.range().start()].contains('\n') { + if token.text().to_string()[edit.delete - token.range().start()].contains('\n') { return None; } } - let text = get_text_after_edit(node, &edit); - let tokens = tokenize(&text); - let token = match tokens[..] { - [token] if token.kind == node.kind() => token, + let text = get_text_after_edit(token.into(), &edit); + let lex_tokens = tokenize(&text); + let lex_token = match lex_tokens[..] { + [lex_token] if lex_token.kind == token.kind() => lex_token, _ => return None, }; - if token.kind == IDENT && is_contextual_kw(&text) { + if lex_token.kind == IDENT && is_contextual_kw(&text) { return None; } - if let Some(next_char) = root.text().char_at(node.range().end()) { + if let Some(next_char) = root.text().char_at(token.range().end()) { let tokens_with_next_char = tokenize(&format!("{}{}", text, next_char)); if tokens_with_next_char.len() == 1 { return None; } } - let green = GreenNode::new_leaf(node.kind(), text.into()); - let new_errors = vec![]; - Some((node, green, new_errors)) + let new_token = GreenToken::new(token.kind(), text.into()); + Some((token.replace_with(new_token), token.range())) } _ => None, } } fn reparse_block<'node>( - node: &'node SyntaxNode, + root: &'node SyntaxNode, edit: &AtomTextEdit, -) -> Option<(&'node SyntaxNode, GreenNode, Vec)> { - let (node, reparser) = find_reparsable_node(node, edit.delete)?; - let text = get_text_after_edit(node, &edit); +) -> Option<(GreenNode, Vec, TextRange)> { + let (node, reparser) = find_reparsable_node(root, edit.delete)?; + let text = get_text_after_edit(node.into(), &edit); let tokens = tokenize(&text); if !is_balanced(&tokens) { return None; @@ -86,12 +88,16 @@ fn reparse_block<'node>( let mut tree_sink = TextTreeSink::new(&text, &tokens); reparser.parse(&token_source, &mut tree_sink); let (green, new_errors) = tree_sink.finish(); - Some((node, green, new_errors)) + Some((node.replace_with(green), new_errors, node.range())) } -fn get_text_after_edit(node: &SyntaxNode, edit: &AtomTextEdit) -> String { - let edit = AtomTextEdit::replace(edit.delete - node.range().start(), edit.insert.clone()); - edit.apply(node.text().to_string()) +fn get_text_after_edit(element: SyntaxElement, edit: &AtomTextEdit) -> String { + let edit = AtomTextEdit::replace(edit.delete - element.range().start(), edit.insert.clone()); + let text = match element { + SyntaxElement::Token(token) => token.text().to_string(), + SyntaxElement::Node(node) => node.text().to_string(), + }; + edit.apply(text) } fn is_contextual_kw(text: &str) -> bool { @@ -102,9 +108,13 @@ fn is_contextual_kw(text: &str) -> bool { } fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, Reparser)> { - let node = algo::find_covering_node(node, range); - node.ancestors().find_map(|node| { - let first_child = node.first_child().map(|it| it.kind()); + let node = algo::find_covering_element(node, range); + let mut ancestors = match node { + SyntaxElement::Token(it) => it.parent().ancestors(), + SyntaxElement::Node(it) => it.ancestors(), + }; + ancestors.find_map(|node| { + let first_child = node.first_child_or_token().map(|it| it.kind()); let parent = node.parent().map(|it| it.kind()); Reparser::for_node(node.kind(), first_child, parent).map(|r| (node, r)) }) @@ -136,19 +146,19 @@ fn is_balanced(tokens: &[Token]) -> bool { fn merge_errors( old_errors: Vec, new_errors: Vec, - old_node: &SyntaxNode, + old_range: TextRange, edit: &AtomTextEdit, ) -> Vec { let mut res = Vec::new(); for e in old_errors { - if e.offset() <= old_node.range().start() { + if e.offset() <= old_range.start() { res.push(e) - } else if e.offset() >= old_node.range().end() { + } else if e.offset() >= old_range.end() { res.push(e.add_offset(TextUnit::of_str(&edit.insert), edit.delete.len())); } } for e in new_errors { - res.push(e.add_offset(old_node.range().start(), 0.into())); + res.push(e.add_offset(old_range.start(), 0.into())); } res } @@ -160,13 +170,7 @@ mod tests { use crate::{SourceFile, AstNode}; use super::*; - fn do_check(before: &str, replace_with: &str, reparser: F) - where - for<'a> F: Fn( - &'a SyntaxNode, - &AtomTextEdit, - ) -> Option<(&'a SyntaxNode, GreenNode, Vec)>, - { + fn do_check(before: &str, replace_with: &str, reparsed_len: u32) { let (range, before) = extract_range(before); let edit = AtomTextEdit::replace(range, replace_with.to_owned()); let after = edit.apply(before.clone()); @@ -175,23 +179,20 @@ mod tests { let incrementally_reparsed = { let f = SourceFile::parse(&before); let edit = AtomTextEdit { delete: range, insert: replace_with.to_string() }; - let (node, green, new_errors) = - reparser(f.syntax(), &edit).expect("cannot incrementally reparse"); - let green_root = node.replace_with(green); - let errors = super::merge_errors(f.errors(), new_errors, node, &edit); - SourceFile::new(green_root, errors) + let (green, new_errors, range) = + incremental_reparse(f.syntax(), &edit, f.errors()).unwrap(); + assert_eq!(range.len(), reparsed_len.into(), "reparsed fragment has wrong length"); + SourceFile::new(green, new_errors) }; assert_eq_text!( &fully_reparsed.syntax().debug_dump(), &incrementally_reparsed.syntax().debug_dump(), - ) + ); } - #[test] + #[test] // FIXME: some test here actually test token reparsing fn reparse_block_tests() { - let do_check = |before, replace_to| do_check(before, replace_to, reparse_block); - do_check( r" fn foo() { @@ -199,6 +200,7 @@ fn foo() { } ", "baz", + 3, ); do_check( r" @@ -207,6 +209,7 @@ fn foo() { } ", "baz", + 25, ); do_check( r" @@ -215,6 +218,7 @@ struct Foo { } ", ",\n g: (),", + 14, ); do_check( r" @@ -225,6 +229,7 @@ fn foo { } ", "62", + 31, // FIXME: reparse only int literal here ); do_check( r" @@ -233,7 +238,9 @@ mod foo { } ", "bar", + 11, ); + do_check( r" trait Foo { @@ -241,6 +248,7 @@ trait Foo { } ", "Output", + 3, ); do_check( r" @@ -249,13 +257,9 @@ impl IntoIterator for Foo { } ", "n next(", + 9, ); - do_check( - r" -use a::b::{foo,<|>,bar<|>}; - ", - "baz", - ); + do_check(r"use a::b::{foo,<|>,bar<|>};", "baz", 10); do_check( r" pub enum A { @@ -263,12 +267,14 @@ pub enum A { } ", "\nBar;\n", + 11, ); do_check( r" foo!{a, b<|><|> d} ", ", c[3]", + 8, ); do_check( r" @@ -277,6 +283,7 @@ fn foo() { } ", "123", + 14, ); do_check( r" @@ -285,54 +292,60 @@ extern { } ", " exit(code: c_int)", + 11, ); } #[test] - fn reparse_leaf_tests() { - let do_check = |before, replace_to| do_check(before, replace_to, reparse_leaf); - + fn reparse_token_tests() { do_check( r"<|><|> fn foo() -> i32 { 1 } ", "\n\n\n \n", + 1, ); do_check( r" fn foo() -> <|><|> {} ", " \n", + 2, ); do_check( r" fn <|>foo<|>() -> i32 { 1 } ", "bar", + 3, ); do_check( r" fn foo<|><|>foo() { } ", "bar", + 6, ); do_check( r" fn foo /* <|><|> */ () {} ", "some comment", + 6, ); do_check( r" fn baz <|><|> () {} ", " \t\t\n\n", + 2, ); do_check( r" fn baz <|><|> () {} ", " \t\t\n\n", + 2, ); do_check( r" @@ -340,24 +353,28 @@ fn baz <|><|> () {} mod { } ", "c", + 14, ); do_check( r#" fn -> &str { "Hello<|><|>" } "#, ", world", + 7, ); do_check( r#" fn -> &str { // "Hello<|><|>" "#, ", world", + 10, ); do_check( r##" fn -> &str { r#"Hello<|><|>"# "##, ", world", + 10, ); do_check( r" @@ -367,6 +384,7 @@ enum Foo { } ", "Clone", + 4, ); } } diff --git a/crates/ra_syntax/src/parsing/text_tree_sink.rs b/crates/ra_syntax/src/parsing/text_tree_sink.rs index b17d06c61..71fc515f2 100644 --- a/crates/ra_syntax/src/parsing/text_tree_sink.rs +++ b/crates/ra_syntax/src/parsing/text_tree_sink.rs @@ -28,10 +28,10 @@ enum State { } impl<'a> TreeSink for TextTreeSink<'a> { - fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { + fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_branch(), + State::PendingFinish => self.inner.finish_node(), State::Normal => (), } self.eat_trivias(); @@ -40,18 +40,18 @@ impl<'a> TreeSink for TextTreeSink<'a> { .iter() .map(|it| it.len) .sum::(); - self.do_leaf(kind, len, n_tokens); + self.do_token(kind, len, n_tokens); } - fn start_branch(&mut self, kind: SyntaxKind) { + fn start_node(&mut self, kind: SyntaxKind) { match mem::replace(&mut self.state, State::Normal) { State::PendingStart => { - self.inner.start_branch(kind); + self.inner.start_node(kind); // No need to attach trivias to previous node: there is no // previous node. return; } - State::PendingFinish => self.inner.finish_branch(), + State::PendingFinish => self.inner.finish_node(), State::Normal => (), } @@ -71,14 +71,14 @@ impl<'a> TreeSink for TextTreeSink<'a> { n_attached_trivias(kind, leading_trivias) }; self.eat_n_trivias(n_trivias - n_attached_trivias); - self.inner.start_branch(kind); + self.inner.start_node(kind); self.eat_n_trivias(n_attached_trivias); } - fn finish_branch(&mut self) { + fn finish_node(&mut self) { match mem::replace(&mut self.state, State::PendingFinish) { State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_branch(), + State::PendingFinish => self.inner.finish_node(), State::Normal => (), } } @@ -104,7 +104,7 @@ impl<'a> TextTreeSink<'a> { match mem::replace(&mut self.state, State::Normal) { State::PendingFinish => { self.eat_trivias(); - self.inner.finish_branch() + self.inner.finish_node() } State::PendingStart | State::Normal => unreachable!(), } @@ -117,7 +117,7 @@ impl<'a> TextTreeSink<'a> { if !token.kind.is_trivia() { break; } - self.do_leaf(token.kind, token.len, 1); + self.do_token(token.kind, token.len, 1); } } @@ -125,16 +125,16 @@ impl<'a> TextTreeSink<'a> { for _ in 0..n { let token = self.tokens[self.token_pos]; assert!(token.kind.is_trivia()); - self.do_leaf(token.kind, token.len, 1); + self.do_token(token.kind, token.len, 1); } } - fn do_leaf(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { + fn do_token(&mut self, kind: SyntaxKind, len: TextUnit, n_tokens: usize) { let range = TextRange::offset_len(self.text_pos, len); let text: SmolStr = self.text[range].into(); self.text_pos += len; self.token_pos += n_tokens; - self.inner.leaf(kind, text); + self.inner.token(kind, text); } } -- cgit v1.2.3