Merge #1078

1078: rewrite syntax trees r=matklad a=matklad



Co-authored-by: Aleksey Kladov <[email protected]>

13 files changed, 274 insertions, 244 deletions

diff --git a/crates/ra_ide_api/src/completion.rs b/crates/ra_ide_api/src/completion.rs
index 639942f7b..a846a7a3c 100644
--- a/crates/ra_ide_api/src/completion.rs
+++ b/crates/ra_ide_api/src/completion.rs
@@ -13,7 +13,7 @@ mod complete_scope;
 mod complete_postfix;
 
 use ra_db::SourceDatabase;
-use ra_syntax::ast::{self, AstNode};
+use ra_syntax::{ast::{self, AstNode}, SyntaxKind::{ATTR, COMMENT}};
 
 use crate::{
     db,
@@ -76,11 +76,10 @@ pub fn function_label(node: &ast::FnDef) -> Option<String> {
         let body_range = body.syntax().range();
         let label: String = node
             .syntax()
-            .children()
+            .children_with_tokens()
             .filter(|child| !child.range().is_subrange(&body_range)) // Filter out body
-            .filter(|child| ast::Comment::cast(child).is_none()) // Filter out comments
-            .filter(|child| ast::Attr::cast(child).is_none()) // Filter out attributes
-            .map(|node| node.text().to_string())
+            .filter(|child| !(child.kind() == COMMENT || child.kind() == ATTR)) // Filter out comments and attrs
+            .map(|node| node.to_string())
             .collect();
         label
     } else {
@@ -93,10 +92,9 @@ pub fn function_label(node: &ast::FnDef) -> Option<String> {
 pub fn const_label(node: &ast::ConstDef) -> String {
     let label: String = node
         .syntax()
-        .children()
-        .filter(|child| ast::Comment::cast(child).is_none())
-        .filter(|child| ast::Attr::cast(child).is_none())
-        .map(|node| node.text().to_string())
+        .children_with_tokens()
+        .filter(|child| !(child.kind() == COMMENT || child.kind() == ATTR))
+        .map(|node| node.to_string())
         .collect();
 
     label.trim().to_owned()
@@ -105,10 +103,9 @@ pub fn const_label(node: &ast::ConstDef) -> String {
 pub fn type_label(node: &ast::TypeAliasDef) -> String {
     let label: String = node
         .syntax()
-        .children()
-        .filter(|child| ast::Comment::cast(child).is_none())
-        .filter(|child| ast::Attr::cast(child).is_none())
-        .map(|node| node.text().to_string())
+        .children_with_tokens()
+        .filter(|child| !(child.kind() == COMMENT || child.kind() == ATTR))
+        .map(|node| node.to_string())
         .collect();
 
     label.trim().to_owned()
diff --git a/crates/ra_ide_api/src/completion/complete_fn_param.rs b/crates/ra_ide_api/src/completion/complete_fn_param.rs
index ffdc744b2..f87ccdeb9 100644
--- a/crates/ra_ide_api/src/completion/complete_fn_param.rs
+++ b/crates/ra_ide_api/src/completion/complete_fn_param.rs
@@ -17,7 +17,7 @@ pub(super) fn complete_fn_param(acc: &mut Completions, ctx: &CompletionContext)
     }
 
     let mut params = FxHashMap::default();
-    for node in ctx.leaf.ancestors() {
+    for node in ctx.token.parent().ancestors() {
         let _ = visitor_ctx(&mut params)
             .visit::<ast::SourceFile, _>(process)
             .visit::<ast::ItemList, _>(process)
diff --git a/crates/ra_ide_api/src/completion/complete_keyword.rs b/crates/ra_ide_api/src/completion/complete_keyword.rs
index 841c0c554..718b83418 100644
--- a/crates/ra_ide_api/src/completion/complete_keyword.rs
+++ b/crates/ra_ide_api/src/completion/complete_keyword.rs
@@ -2,7 +2,7 @@ use ra_syntax::{
     algo::visit::{visitor, Visitor},
     AstNode,
     ast::{self, LoopBodyOwner},
-    SyntaxKind::*, SyntaxNode,
+    SyntaxKind::*, SyntaxToken,
 };
 
 use crate::completion::{CompletionContext, CompletionItem, Completions, CompletionKind, CompletionItemKind};
@@ -62,7 +62,7 @@ pub(super) fn complete_expr_keyword(acc: &mut Completions, ctx: &CompletionConte
         acc.add(keyword(ctx, "else", "else {$0}"));
         acc.add(keyword(ctx, "else if", "else if $0 {}"));
     }
-    if is_in_loop_body(ctx.leaf) {
+    if is_in_loop_body(ctx.token) {
         if ctx.can_be_stmt {
             acc.add(keyword(ctx, "continue", "continue;"));
             acc.add(keyword(ctx, "break", "break;"));
@@ -74,8 +74,8 @@ pub(super) fn complete_expr_keyword(acc: &mut Completions, ctx: &CompletionConte
     acc.add_all(complete_return(ctx, fn_def, ctx.can_be_stmt));
 }
 
-fn is_in_loop_body(leaf: &SyntaxNode) -> bool {
-    for node in leaf.ancestors() {
+fn is_in_loop_body(leaf: SyntaxToken) -> bool {
+    for node in leaf.parent().ancestors() {
         if node.kind() == FN_DEF || node.kind() == LAMBDA_EXPR {
             break;
         }
diff --git a/crates/ra_ide_api/src/completion/completion_context.rs b/crates/ra_ide_api/src/completion/completion_context.rs
index 724d0dfbf..65dffa470 100644
--- a/crates/ra_ide_api/src/completion/completion_context.rs
+++ b/crates/ra_ide_api/src/completion/completion_context.rs
@@ -1,8 +1,8 @@
 use ra_text_edit::AtomTextEdit;
 use ra_syntax::{
-    AstNode, SyntaxNode, SourceFile, TextUnit, TextRange,
+    AstNode, SyntaxNode, SourceFile, TextUnit, TextRange, SyntaxToken,
     ast,
-    algo::{find_leaf_at_offset, find_covering_node, find_node_at_offset},
+    algo::{find_token_at_offset, find_covering_element, find_node_at_offset},
     SyntaxKind::*,
 };
 use hir::{source_binder, Resolver};
@@ -15,7 +15,7 @@ use crate::{db, FilePosition};
 pub(crate) struct CompletionContext<'a> {
     pub(super) db: &'a db::RootDatabase,
     pub(super) offset: TextUnit,
-    pub(super) leaf: &'a SyntaxNode,
+    pub(super) token: SyntaxToken<'a>,
     pub(super) resolver: Resolver,
     pub(super) module: Option<hir::Module>,
     pub(super) function: Option<hir::Function>,
@@ -49,10 +49,10 @@ impl<'a> CompletionContext<'a> {
     ) -> Option<CompletionContext<'a>> {
         let resolver = source_binder::resolver_for_position(db, position);
         let module = source_binder::module_from_position(db, position);
-        let leaf = find_leaf_at_offset(original_file.syntax(), position.offset).left_biased()?;
+        let token = find_token_at_offset(original_file.syntax(), position.offset).left_biased()?;
         let mut ctx = CompletionContext {
             db,
-            leaf,
+            token,
             offset: position.offset,
             resolver,
             module,
@@ -76,9 +76,9 @@ impl<'a> CompletionContext<'a> {
 
     // The range of the identifier that is being completed.
     pub(crate) fn source_range(&self) -> TextRange {
-        match self.leaf.kind() {
+        match self.token.kind() {
             // workaroud when completion is triggered by trigger characters.
-            IDENT => self.leaf.range(),
+            IDENT => self.token.range(),
             _ => TextRange::offset_len(self.offset, 0.into()),
         }
     }
@@ -139,10 +139,11 @@ impl<'a> CompletionContext<'a> {
             _ => (),
         }
 
-        self.use_item_syntax = self.leaf.ancestors().find_map(ast::UseItem::cast);
+        self.use_item_syntax = self.token.parent().ancestors().find_map(ast::UseItem::cast);
 
         self.function_syntax = self
-            .leaf
+            .token
+            .parent()
             .ancestors()
             .take_while(|it| it.kind() != SOURCE_FILE && it.kind() != MODULE)
             .find_map(ast::FnDef::cast);
@@ -224,8 +225,7 @@ impl<'a> CompletionContext<'a> {
 }
 
 fn find_node_with_range<N: AstNode>(syntax: &SyntaxNode, range: TextRange) -> Option<&N> {
-    let node = find_covering_node(syntax, range);
-    node.ancestors().find_map(N::cast)
+    find_covering_element(syntax, range).ancestors().find_map(N::cast)
 }
 
 fn is_node<N: AstNode>(node: &SyntaxNode) -> bool {
diff --git a/crates/ra_ide_api/src/diagnostics.rs b/crates/ra_ide_api/src/diagnostics.rs
index 5a78e94d8..2dfaa0045 100644
--- a/crates/ra_ide_api/src/diagnostics.rs
+++ b/crates/ra_ide_api/src/diagnostics.rs
@@ -106,8 +106,10 @@ fn text_edit_for_remove_unnecessary_braces_with_self_in_use_statement(
     single_use_tree: &ast::UseTree,
 ) -> Option<TextEdit> {
     let use_tree_list_node = single_use_tree.syntax().parent()?;
-    if single_use_tree.path()?.segment()?.syntax().first_child()?.kind() == SyntaxKind::SELF_KW {
-        let start = use_tree_list_node.prev_sibling()?.range().start();
+    if single_use_tree.path()?.segment()?.syntax().first_child_or_token()?.kind()
+        == SyntaxKind::SELF_KW
+    {
+        let start = use_tree_list_node.prev_sibling_or_token()?.range().start();
         let end = use_tree_list_node.range().end();
         let range = TextRange::from_to(start, end);
         let mut edit_builder = TextEditBuilder::default();
diff --git a/crates/ra_ide_api/src/extend_selection.rs b/crates/ra_ide_api/src/extend_selection.rs
index 63879a0b5..e743bf0fe 100644
--- a/crates/ra_ide_api/src/extend_selection.rs
+++ b/crates/ra_ide_api/src/extend_selection.rs
@@ -1,8 +1,9 @@
 use ra_db::SourceDatabase;
 use ra_syntax::{
-    Direction, SyntaxNode, TextRange, TextUnit, AstNode,
-    algo::{find_covering_node, find_leaf_at_offset, LeafAtOffset},
-    SyntaxKind::*,
+    Direction, SyntaxNode, TextRange, TextUnit, AstNode, SyntaxElement,
+    algo::{find_covering_element, find_token_at_offset, TokenAtOffset},
+    SyntaxKind::*, SyntaxToken,
+    ast::Comment,
 };
 
 use crate::{FileRange, db::RootDatabase};
@@ -32,53 +33,58 @@ fn try_extend_selection(root: &SyntaxNode, range: TextRange) -> Option<TextRange
 
     if range.is_empty() {
         let offset = range.start();
-        let mut leaves = find_leaf_at_offset(root, offset);
+        let mut leaves = find_token_at_offset(root, offset);
         if leaves.clone().all(|it| it.kind() == WHITESPACE) {
             return Some(extend_ws(root, leaves.next()?, offset));
         }
         let leaf_range = match leaves {
-            LeafAtOffset::None => return None,
-            LeafAtOffset::Single(l) => {
+            TokenAtOffset::None => return None,
+            TokenAtOffset::Single(l) => {
                 if string_kinds.contains(&l.kind()) {
                     extend_single_word_in_comment_or_string(l, offset).unwrap_or_else(|| l.range())
                 } else {
                     l.range()
                 }
             }
-            LeafAtOffset::Between(l, r) => pick_best(l, r).range(),
+            TokenAtOffset::Between(l, r) => pick_best(l, r).range(),
         };
         return Some(leaf_range);
     };
-    let node = find_covering_node(root, range);
+    let node = match find_covering_element(root, range) {
+        SyntaxElement::Token(token) => {
+            if token.range() != range {
+                return Some(token.range());
+            }
+            if let Some(comment) = Comment::cast(token) {
+                if let Some(range) = extend_comments(comment) {
+                    return Some(range);
+                }
+            }
+            token.parent()
+        }
+        SyntaxElement::Node(node) => node,
+    };
+    if node.range() != range {
+        return Some(node.range());
+    }
 
     // Using shallowest node with same range allows us to traverse siblings.
     let node = node.ancestors().take_while(|n| n.range() == node.range()).last().unwrap();
 
-    if range == node.range() {
-        if string_kinds.contains(&node.kind()) {
-            if let Some(range) = extend_comments(node) {
-                return Some(range);
-            }
-        }
-
-        if node.parent().map(|n| list_kinds.contains(&n.kind())) == Some(true) {
-            if let Some(range) = extend_list_item(node) {
-                return Some(range);
-            }
+    if node.parent().map(|n| list_kinds.contains(&n.kind())) == Some(true) {
+        if let Some(range) = extend_list_item(node) {
+            return Some(range);
         }
     }
 
-    match node.ancestors().skip_while(|n| n.range() == range).next() {
-        None => None,
-        Some(parent) => Some(parent.range()),
-    }
+    node.parent().map(|it| it.range())
 }
 
 fn extend_single_word_in_comment_or_string(
-    leaf: &SyntaxNode,
+    leaf: SyntaxToken,
     offset: TextUnit,
 ) -> Option<TextRange> {
-    let text: &str = leaf.leaf_text()?;
+    let text: &str = leaf.text();
     let cursor_position: u32 = (offset - leaf.range().start()).into();
 
     let (before, after) = text.split_at(cursor_position as usize);
@@ -101,14 +107,14 @@ fn extend_single_word_in_comment_or_string(
     }
 }
 
-fn extend_ws(root: &SyntaxNode, ws: &SyntaxNode, offset: TextUnit) -> TextRange {
-    let ws_text = ws.leaf_text().unwrap();
+fn extend_ws(root: &SyntaxNode, ws: SyntaxToken, offset: TextUnit) -> TextRange {
+    let ws_text = ws.text();
     let suffix = TextRange::from_to(offset, ws.range().end()) - ws.range().start();
     let prefix = TextRange::from_to(ws.range().start(), offset) - ws.range().start();
     let ws_suffix = &ws_text.as_str()[suffix];
     let ws_prefix = &ws_text.as_str()[prefix];
     if ws_text.contains('\n') && !ws_suffix.contains('\n') {
-        if let Some(node) = ws.next_sibling() {
+        if let Some(node) = ws.next_sibling_or_token() {
             let start = match ws_prefix.rfind('\n') {
                 Some(idx) => ws.range().start() + TextUnit::from((idx + 1) as u32),
                 None => node.range().start(),
@@ -124,9 +130,9 @@ fn extend_ws(root: &SyntaxNode, ws: &SyntaxNode, offset: TextUnit) -> TextRange
     ws.range()
 }
 
-fn pick_best<'a>(l: &'a SyntaxNode, r: &'a SyntaxNode) -> &'a SyntaxNode {
+fn pick_best<'a>(l: SyntaxToken<'a>, r: SyntaxToken<'a>) -> SyntaxToken<'a> {
     return if priority(r) > priority(l) { r } else { l };
-    fn priority(n: &SyntaxNode) -> usize {
+    fn priority(n: SyntaxToken) -> usize {
         match n.kind() {
             WHITESPACE => 0,
             IDENT | SELF_KW | SUPER_KW | CRATE_KW | LIFETIME => 2,
@@ -137,54 +143,60 @@ fn pick_best<'a>(l: &'a SyntaxNode, r: &'a SyntaxNode) -> &'a SyntaxNode {
 
 /// Extend list item selection to include nearby comma and whitespace.
 fn extend_list_item(node: &SyntaxNode) -> Option<TextRange> {
-    fn is_single_line_ws(node: &SyntaxNode) -> bool {
-        node.kind() == WHITESPACE && !node.leaf_text().unwrap().contains('\n')
+    fn is_single_line_ws(node: &SyntaxToken) -> bool {
+        node.kind() == WHITESPACE && !node.text().contains('\n')
     }
 
-    fn nearby_comma(node: &SyntaxNode, dir: Direction) -> Option<&SyntaxNode> {
-        node.siblings(dir)
+    fn nearby_comma(node: &SyntaxNode, dir: Direction) -> Option<SyntaxToken> {
+        node.siblings_with_tokens(dir)
             .skip(1)
-            .skip_while(|node| is_single_line_ws(node))
+            .skip_while(|node| match node {
+                SyntaxElement::Node(_) => false,
+                SyntaxElement::Token(it) => is_single_line_ws(it),
+            })
             .next()
+            .and_then(|it| it.as_token())
             .filter(|node| node.kind() == COMMA)
     }
 
     if let Some(comma_node) = nearby_comma(node, Direction::Prev) {
         return Some(TextRange::from_to(comma_node.range().start(), node.range().end()));
     }
-
     if let Some(comma_node) = nearby_comma(node, Direction::Next) {
         // Include any following whitespace when comma if after list item.
         let final_node = comma_node
-            .siblings(Direction::Next)
-            .skip(1)
-            .next()
+            .next_sibling_or_token()
+            .and_then(|it| it.as_token())
             .filter(|node| is_single_line_ws(node))
             .unwrap_or(comma_node);
 
         return Some(TextRange::from_to(node.range().start(), final_node.range().end()));
     }
 
-    return None;
+    None
 }
 
-fn extend_comments(node: &SyntaxNode) -> Option<TextRange> {
-    let prev = adj_comments(node, Direction::Prev);
-    let next = adj_comments(node, Direction::Next);
+fn extend_comments(comment: Comment) -> Option<TextRange> {
+    let prev = adj_comments(comment, Direction::Prev);
+    let next = adj_comments(comment, Direction::Next);
     if prev != next {
-        Some(TextRange::from_to(prev.range().start(), next.range().end()))
+        Some(TextRange::from_to(prev.syntax().range().start(), next.syntax().range().end()))
     } else {
         None
     }
 }
 
-fn adj_comments(node: &SyntaxNode, dir: Direction) -> &SyntaxNode {
-    let mut res = node;
-    for node in node.siblings(dir) {
-        match node.kind() {
-            COMMENT => res = node,
-            WHITESPACE if !node.leaf_text().unwrap().as_str().contains("\n\n") => (),
-            _ => break,
+fn adj_comments(comment: Comment, dir: Direction) -> Comment {
+    let mut res = comment;
+    for element in comment.syntax().siblings_with_tokens(dir) {
+        let token = match element.as_token() {
+            None => break,
+            Some(token) => token,
+        };
+        if let Some(c) = Comment::cast(token) {
+            res = c
+        } else if token.kind() != WHITESPACE || token.text().contains("\n\n") {
+            break;
         }
     }
     res
@@ -308,23 +320,13 @@ fn bar(){}
 /*
 foo
 _bar1<|>*/
-    "#,
+"#,
             &["_bar1", "/*\nfoo\n_bar1*/"],
         );
 
-        do_check(
-            r#"
-//!<|>foo_2 bar
-    "#,
-            &["foo_2", "//!foo_2 bar"],
-        );
+        do_check(r#"//!<|>foo_2 bar"#, &["foo_2", "//!foo_2 bar"]);
 
-        do_check(
-            r#"
-/<|>/foo bar
-    "#,
-            &["//foo bar"],
-        );
+        do_check(r#"/<|>/foo bar"#, &["//foo bar"]);
     }
 
     #[test]
@@ -332,13 +334,13 @@ _bar1<|>*/
         do_check(
             r#"
 fn main() { foo<|>+bar;}
-    "#,
+"#,
             &["foo", "foo+bar"],
         );
         do_check(
             r#"
 fn main() { foo+<|>bar;}
-    "#,
+"#,
             &["bar", "foo+bar"],
         );
     }
@@ -355,11 +357,11 @@ fn main() { foo+<|>bar;}
         do_check(
             r#"
 impl S {
-    fn foo() {
-        // hel<|>lo world
-    }
+fn foo() {
+// hel<|>lo world
 }
-        "#,
+}
+"#,
             &["hello", "// hello world"],
         );
     }
@@ -371,7 +373,7 @@ impl S {
 fn bar(){}
 
 " fn f<|>oo() {"
-    "#,
+"#,
             &["foo", "\" fn foo() {\""],
         );
     }
diff --git a/crates/ra_ide_api/src/folding_ranges.rs b/crates/ra_ide_api/src/folding_ranges.rs
index b96145f05..a6fe8a5d5 100644
--- a/crates/ra_ide_api/src/folding_ranges.rs
+++ b/crates/ra_ide_api/src/folding_ranges.rs
@@ -1,9 +1,9 @@
 use rustc_hash::FxHashSet;
 
 use ra_syntax::{
-    AstNode, Direction, SourceFile, SyntaxNode, TextRange,
+    AstNode, SourceFile, SyntaxNode, TextRange, Direction, SyntaxElement,
     SyntaxKind::{self, *},
-    ast::{self, VisibilityOwner},
+    ast::{self, VisibilityOwner, Comment},
 };
 
 #[derive(Debug, PartialEq, Eq)]
@@ -26,34 +26,49 @@ pub(crate) fn folding_ranges(file: &SourceFile) -> Vec<Fold> {
     let mut visited_imports = FxHashSet::default();
     let mut visited_mods = FxHashSet::default();
 
-    for node in file.syntax().descendants() {
+    for element in file.syntax().descendants_with_tokens() {
         // Fold items that span multiple lines
-        if let Some(kind) = fold_kind(node.kind()) {
-            if node.text().contains('\n') {
-                res.push(Fold { range: node.range(), kind });
+        if let Some(kind) = fold_kind(element.kind()) {
+            let is_multiline = match element {
+                SyntaxElement::Node(node) => node.text().contains('\n'),
+                SyntaxElement::Token(token) => token.text().contains('\n'),
+            };
+            if is_multiline {
+                res.push(Fold { range: element.range(), kind });
+                continue;
             }
         }
 
-        // Fold groups of comments
-        if node.kind() == COMMENT && !visited_comments.contains(&node) {
-            if let Some(range) = contiguous_range_for_comment(node, &mut visited_comments) {
-                res.push(Fold { range, kind: FoldKind::Comment })
+        match element {
+            SyntaxElement::Token(token) => {
+                // Fold groups of comments
+                if let Some(comment) = ast::Comment::cast(token) {
+                    if !visited_comments.contains(&comment) {
+                        if let Some(range) =
+                            contiguous_range_for_comment(comment, &mut visited_comments)
+                        {
+                            res.push(Fold { range, kind: FoldKind::Comment })
+                        }
+                    }
+                }
             }
-        }
-
-        // Fold groups of imports
-        if node.kind() == USE_ITEM && !visited_imports.contains(&node) {
-            if let Some(range) = contiguous_range_for_group(node, &mut visited_imports) {
-                res.push(Fold { range, kind: FoldKind::Imports })
-            }
-        }
-
-        // Fold groups of mods
-        if node.kind() == MODULE && !has_visibility(&node) && !visited_mods.contains(&node) {
-            if let Some(range) =
-                contiguous_range_for_group_unless(node, has_visibility, &mut visited_mods)
-            {
-                res.push(Fold { range, kind: FoldKind::Mods })
+            SyntaxElement::Node(node) => {
+                // Fold groups of imports
+                if node.kind() == USE_ITEM && !visited_imports.contains(&node) {
+                    if let Some(range) = contiguous_range_for_group(node, &mut visited_imports) {
+                        res.push(Fold { range, kind: FoldKind::Imports })
+                    }
+                }
+
+                // Fold groups of mods
+                if node.kind() == MODULE && !has_visibility(&node) && !visited_mods.contains(&node)
+                {
+                    if let Some(range) =
+                        contiguous_range_for_group_unless(node, has_visibility, &mut visited_mods)
+                    {
+                        res.push(Fold { range, kind: FoldKind::Mods })
+                    }
+                }
             }
         }
     }
@@ -90,16 +105,21 @@ fn contiguous_range_for_group_unless<'a>(
     visited.insert(first);
 
     let mut last = first;
-    for node in first.siblings(Direction::Next) {
-        if let Some(ws) = ast::Whitespace::cast(node) {
-            // There is a blank line, which means that the group ends here
-            if ws.count_newlines_lazy().take(2).count() == 2 {
+    for element in first.siblings_with_tokens(Direction::Next) {
+        let node = match element {
+            SyntaxElement::Token(token) => {
+                if let Some(ws) = ast::Whitespace::cast(token) {
+                    if !ws.spans_multiple_lines() {
+                        // Ignore whitespace without blank lines
+                        continue;
+                    }
+                }
+                // There is a blank line or another token, which means that the
+                // group ends here
                 break;
             }
-
-            // Ignore whitespace without blank lines
-            continue;
-        }
+            SyntaxElement::Node(node) => node,
+        };
 
         // Stop if we find a node that doesn't belong to the group
         if node.kind() != first.kind() || unless(node) {
@@ -119,40 +139,42 @@ fn contiguous_range_for_group_unless<'a>(
 }
 
 fn contiguous_range_for_comment<'a>(
-    first: &'a SyntaxNode,
-    visited: &mut FxHashSet<&'a SyntaxNode>,
+    first: Comment<'a>,
+    visited: &mut FxHashSet<Comment<'a>>,
 ) -> Option<TextRange> {
     visited.insert(first);
 
     // Only fold comments of the same flavor
-    let group_flavor = ast::Comment::cast(first)?.flavor();
+    let group_flavor = first.flavor();
 
     let mut last = first;
-    for node in first.siblings(Direction::Next) {
-        if let Some(ws) = ast::Whitespace::cast(node) {
-            // There is a blank line, which means the group ends here
-            if ws.count_newlines_lazy().take(2).count() == 2 {
+    for element in first.syntax().siblings_with_tokens(Direction::Next) {
+        match element {
+            SyntaxElement::Token(token) => {
+                if let Some(ws) = ast::Whitespace::cast(token) {
+                    if !ws.spans_multiple_lines() {
+                        // Ignore whitespace without blank lines
+                        continue;
+                    }
+                }
+                if let Some(c) = Comment::cast(token) {
+                    if c.flavor() == group_flavor {
+                        visited.insert(c);
+                        last = c;
+                        continue;
+                    }
+                }
+                // The comment group ends because either:
+                // * An element of a different kind was reached
+                // * A comment of a different flavor was reached
                 break;
             }
-
-            // Ignore whitespace without blank lines
-            continue;
-        }
-
-        match ast::Comment::cast(node) {
-            Some(next_comment) if next_comment.flavor() == group_flavor => {
-                visited.insert(node);
-                last = node;
-            }
-            // The comment group ends because either:
-            // * An element of a different kind was reached
-            // * A comment of a different flavor was reached
-            _ => break,
-        }
+            SyntaxElement::Node(_) => break,
+        };
     }
 
     if first != last {
-        Some(TextRange::from_to(first.range().start(), last.range().end()))
+        Some(TextRange::from_to(first.syntax().range().start(), last.syntax().range().end()))
     } else {
         // The group consists of only one element, therefore it cannot be folded
         None
diff --git a/crates/ra_ide_api/src/hover.rs b/crates/ra_ide_api/src/hover.rs
index 3206e68b9..bfa7cd67a 100644
--- a/crates/ra_ide_api/src/hover.rs
+++ b/crates/ra_ide_api/src/hover.rs
@@ -1,7 +1,7 @@
 use ra_db::SourceDatabase;
 use ra_syntax::{
     AstNode, SyntaxNode, TreeArc, ast::{self, NameOwner, VisibilityOwner, TypeAscriptionOwner},
-    algo::{find_covering_node, find_node_at_offset, find_leaf_at_offset, visit::{visitor, Visitor}},
+    algo::{find_covering_element, find_node_at_offset, find_token_at_offset, visit::{visitor, Visitor}},
 };
 use hir::HirDisplay;
 
@@ -104,8 +104,11 @@ pub(crate) fn hover(db: &RootDatabase, position: FilePosition) -> Option<RangeIn
     }
 
     if range.is_none() {
-        let node = find_leaf_at_offset(file.syntax(), position.offset).find_map(|leaf| {
-            leaf.ancestors().find(|n| ast::Expr::cast(*n).is_some() || ast::Pat::cast(*n).is_some())
+        let node = find_token_at_offset(file.syntax(), position.offset).find_map(|token| {
+            token
+                .parent()
+                .ancestors()
+                .find(|n| ast::Expr::cast(*n).is_some() || ast::Pat::cast(*n).is_some())
         })?;
         let frange = FileRange { file_id: position.file_id, range: node.range() };
         res.extend(type_of(db, frange).map(rust_code_markup));
@@ -123,13 +126,12 @@ pub(crate) fn hover(db: &RootDatabase, position: FilePosition) -> Option<RangeIn
 pub(crate) fn type_of(db: &RootDatabase, frange: FileRange) -> Option<String> {
     let file = db.parse(frange.file_id);
     let syntax = file.syntax();
-    let leaf_node = find_covering_node(syntax, frange.range);
+    let leaf_node = find_covering_element(syntax, frange.range);
     // if we picked identifier, expand to pattern/expression
     let node = leaf_node
         .ancestors()
         .take_while(|it| it.range() == leaf_node.range())
-        .find(|&it| ast::Expr::cast(it).is_some() || ast::Pat::cast(it).is_some())
-        .unwrap_or(leaf_node);
+        .find(|&it| ast::Expr::cast(it).is_some() || ast::Pat::cast(it).is_some())?;
     let parent_fn = node.ancestors().find_map(ast::FnDef::cast)?;
     let function = hir::source_binder::function_from_source(db, frange.file_id, parent_fn)?;
     let infer = function.infer(db);
diff --git a/crates/ra_ide_api/src/join_lines.rs b/crates/ra_ide_api/src/join_lines.rs
index 8fb3eaa06..57b6f8384 100644
--- a/crates/ra_ide_api/src/join_lines.rs
+++ b/crates/ra_ide_api/src/join_lines.rs
@@ -1,8 +1,8 @@
 use itertools::Itertools;
 use ra_syntax::{
-    SourceFile, TextRange, TextUnit, AstNode, SyntaxNode,
+    SourceFile, TextRange, TextUnit, AstNode, SyntaxNode, SyntaxElement, SyntaxToken,
     SyntaxKind::{self, WHITESPACE, COMMA, R_CURLY, R_PAREN, R_BRACK},
-    algo::{find_covering_node, non_trivia_sibling},
+    algo::{find_covering_element, non_trivia_sibling},
     ast,
     Direction,
 };
@@ -24,22 +24,22 @@ pub fn join_lines(file: &SourceFile, range: TextRange) -> TextEdit {
         range
     };
 
-    let node = find_covering_node(file.syntax(), range);
+    let node = match find_covering_element(file.syntax(), range) {
+        SyntaxElement::Node(node) => node,
+        SyntaxElement::Token(token) => token.parent(),
+    };
     let mut edit = TextEditBuilder::default();
-    for node in node.descendants() {
-        let text = match node.leaf_text() {
-            Some(text) => text,
-            None => continue,
-        };
-        let range = match range.intersection(&node.range()) {
+    for token in node.descendants_with_tokens().filter_map(|it| it.as_token()) {
+        let range = match range.intersection(&token.range()) {
             Some(range) => range,
             None => continue,
-        } - node.range().start();
+        } - token.range().start();
+        let text = token.text();
         for (pos, _) in text[range].bytes().enumerate().filter(|&(_, b)| b == b'\n') {
             let pos: TextUnit = (pos as u32).into();
-            let off = node.range().start() + range.start() + pos;
+            let off = token.range().start() + range.start() + pos;
             if !edit.invalidates_offset(off) {
-                remove_newline(&mut edit, node, text.as_str(), off);
+                remove_newline(&mut edit, token, off);
             }
         }
     }
@@ -47,17 +47,12 @@ pub fn join_lines(file: &SourceFile, range: TextRange) -> TextEdit {
     edit.finish()
 }
 
-fn remove_newline(
-    edit: &mut TextEditBuilder,
-    node: &SyntaxNode,
-    node_text: &str,
-    offset: TextUnit,
-) {
-    if node.kind() != WHITESPACE || node_text.bytes().filter(|&b| b == b'\n').count() != 1 {
+fn remove_newline(edit: &mut TextEditBuilder, token: SyntaxToken, offset: TextUnit) {
+    if token.kind() != WHITESPACE || token.text().bytes().filter(|&b| b == b'\n').count() != 1 {
         // The node is either the first or the last in the file
-        let suff = &node_text[TextRange::from_to(
-            offset - node.range().start() + TextUnit::of_char('\n'),
-            TextUnit::of_str(node_text),
+        let suff = &token.text()[TextRange::from_to(
+            offset - token.range().start() + TextUnit::of_char('\n'),
+            TextUnit::of_str(token.text()),
         )];
         let spaces = suff.bytes().take_while(|&b| b == b' ').count();
 
@@ -74,7 +69,7 @@ fn remove_newline(
     // ```
     //
     // into `my_function(<some-expr>)`
-    if join_single_expr_block(edit, node).is_some() {
+    if join_single_expr_block(edit, token).is_some() {
         return;
     }
     // ditto for
@@ -84,44 +79,50 @@ fn remove_newline(
     //    bar
     // };
     // ```
-    if join_single_use_tree(edit, node).is_some() {
+    if join_single_use_tree(edit, token).is_some() {
         return;
     }
 
     // The node is between two other nodes
-    let prev = node.prev_sibling().unwrap();
-    let next = node.next_sibling().unwrap();
+    let prev = token.prev_sibling_or_token().unwrap();
+    let next = token.next_sibling_or_token().unwrap();
     if is_trailing_comma(prev.kind(), next.kind()) {
         // Removes: trailing comma, newline (incl. surrounding whitespace)
-        edit.delete(TextRange::from_to(prev.range().start(), node.range().end()));
+        edit.delete(TextRange::from_to(prev.range().start(), token.range().end()));
     } else if prev.kind() == COMMA && next.kind() == R_CURLY {
         // Removes: comma, newline (incl. surrounding whitespace)
-        let space = if let Some(left) = prev.prev_sibling() { compute_ws(left, next) } else { " " };
+        let space = if let Some(left) = prev.prev_sibling_or_token() {
+            compute_ws(left.kind(), next.kind())
+        } else {
+            " "
+        };
         edit.replace(
-            TextRange::from_to(prev.range().start(), node.range().end()),
+            TextRange::from_to(prev.range().start(), token.range().end()),
             space.to_string(),
         );
-    } else if let (Some(_), Some(next)) = (ast::Comment::cast(prev), ast::Comment::cast(next)) {
+    } else if let (Some(_), Some(next)) =
+        (prev.as_token().and_then(ast::Comment::cast), next.as_token().and_then(ast::Comment::cast))
+    {
         // Removes: newline (incl. surrounding whitespace), start of the next comment
         edit.delete(TextRange::from_to(
-            node.range().start(),
+            token.range().start(),
             next.syntax().range().start() + TextUnit::of_str(next.prefix()),
         ));
     } else {
         // Remove newline but add a computed amount of whitespace characters
-        edit.replace(node.range(), compute_ws(prev, next).to_string());
+        edit.replace(token.range(), compute_ws(prev.kind(), next.kind()).to_string());
     }
 }
 
 fn has_comma_after(node: &SyntaxNode) -> bool {
-    match non_trivia_sibling(node, Direction::Next) {
+    match non_trivia_sibling(node.into(), Direction::Next) {
         Some(n) => n.kind() == COMMA,
         _ => false,
     }
 }
 
-fn join_single_expr_block(edit: &mut TextEditBuilder, node: &SyntaxNode) -> Option<()> {
-    let block = ast::Block::cast(node.parent()?)?;
+fn join_single_expr_block(edit: &mut TextEditBuilder, token: SyntaxToken) -> Option<()> {
+    let block = ast::Block::cast(token.parent())?;
     let block_expr = ast::BlockExpr::cast(block.syntax().parent()?)?;
     let expr = extract_trivial_expression(block)?;
 
@@ -140,8 +141,8 @@ fn join_single_expr_block(edit: &mut TextEditBuilder, node: &SyntaxNode) -> Opti
     Some(())
 }
 
-fn join_single_use_tree(edit: &mut TextEditBuilder, node: &SyntaxNode) -> Option<()> {
-    let use_tree_list = ast::UseTreeList::cast(node.parent()?)?;
+fn join_single_use_tree(edit: &mut TextEditBuilder, token: SyntaxToken) -> Option<()> {
+    let use_tree_list = ast::UseTreeList::cast(token.parent())?;
     let (tree,) = use_tree_list.use_trees().collect_tuple()?;
     edit.replace(use_tree_list.syntax().range(), tree.syntax().text().to_string());
     Some(())
@@ -401,13 +402,13 @@ use ra_syntax::{
             r"
 use ra_syntax::{
     algo::<|>{
-        find_leaf_at_offset,
+        find_token_at_offset,
     },
     ast,
 };",
             r"
 use ra_syntax::{
-    algo::<|>find_leaf_at_offset,
+    algo::<|>find_token_at_offset,
     ast,
 };",
         );
diff --git a/crates/ra_ide_api/src/matching_brace.rs b/crates/ra_ide_api/src/matching_brace.rs
index d1405f14f..bebd16a69 100644
--- a/crates/ra_ide_api/src/matching_brace.rs
+++ b/crates/ra_ide_api/src/matching_brace.rs
@@ -1,6 +1,6 @@
 use ra_syntax::{
     SourceFile, TextUnit,
-    algo::find_leaf_at_offset,
+    algo::find_token_at_offset,
     SyntaxKind::{self, *},
     ast::AstNode,
 };
@@ -8,15 +8,15 @@ use ra_syntax::{
 pub fn matching_brace(file: &SourceFile, offset: TextUnit) -> Option<TextUnit> {
     const BRACES: &[SyntaxKind] =
         &[L_CURLY, R_CURLY, L_BRACK, R_BRACK, L_PAREN, R_PAREN, L_ANGLE, R_ANGLE];
-    let (brace_node, brace_idx) = find_leaf_at_offset(file.syntax(), offset)
+    let (brace_node, brace_idx) = find_token_at_offset(file.syntax(), offset)
         .filter_map(|node| {
             let idx = BRACES.iter().position(|&brace| brace == node.kind())?;
             Some((node, idx))
         })
         .next()?;
-    let parent = brace_node.parent()?;
+    let parent = brace_node.parent();
     let matching_kind = BRACES[brace_idx ^ 1];
-    let matching_node = parent.children().find(|node| node.kind() == matching_kind)?;
+    let matching_node = parent.children_with_tokens().find(|node| node.kind() == matching_kind)?;
     Some(matching_node.range().start())
 }
 
@@ -41,5 +41,4 @@ mod tests {
 
         do_check("struct Foo { a: i32, }<|>", "struct Foo <|>{ a: i32, }");
     }
-
 }
diff --git a/crates/ra_ide_api/src/syntax_highlighting.rs b/crates/ra_ide_api/src/syntax_highlighting.rs
index a0c5e78ad..d9a28d2b5 100644
--- a/crates/ra_ide_api/src/syntax_highlighting.rs
+++ b/crates/ra_ide_api/src/syntax_highlighting.rs
@@ -1,6 +1,6 @@
 use rustc_hash::FxHashSet;
 
-use ra_syntax::{ast, AstNode, TextRange, Direction, SyntaxKind::*};
+use ra_syntax::{ast, AstNode, TextRange, Direction, SyntaxKind::*, SyntaxElement};
 use ra_db::SourceDatabase;
 
 use crate::{FileId, db::RootDatabase};
@@ -15,9 +15,9 @@ pub(crate) fn highlight(db: &RootDatabase, file_id: FileId) -> Vec<HighlightedRa
     let source_file = db.parse(file_id);
 
     // Visited nodes to handle highlighting priorities
-    let mut highlighted = FxHashSet::default();
+    let mut highlighted: FxHashSet<SyntaxElement> = FxHashSet::default();
     let mut res = Vec::new();
-    for node in source_file.syntax().descendants() {
+    for node in source_file.syntax().descendants_with_tokens() {
         if highlighted.contains(&node) {
             continue;
         }
@@ -31,14 +31,14 @@ pub(crate) fn highlight(db: &RootDatabase, file_id: FileId) -> Vec<HighlightedRa
             LIFETIME => "parameter",
             k if k.is_keyword() => "keyword",
             _ => {
-                if let Some(macro_call) = ast::MacroCall::cast(node) {
+                if let Some(macro_call) = node.as_node().and_then(ast::MacroCall::cast) {
                     if let Some(path) = macro_call.path() {
                         if let Some(segment) = path.segment() {
                             if let Some(name_ref) = segment.name_ref() {
-                                highlighted.insert(name_ref.syntax());
+                                highlighted.insert(name_ref.syntax().into());
                                 let range_start = name_ref.syntax().range().start();
                                 let mut range_end = name_ref.syntax().range().end();
-                                for sibling in path.syntax().siblings(Direction::Next) {
+                                for sibling in path.syntax().siblings_with_tokens(Direction::Next) {
                                     match sibling.kind() {
                                         EXCL | IDENT => range_end = sibling.range().end(),
                                         _ => (),
diff --git a/crates/ra_ide_api/src/syntax_tree.rs b/crates/ra_ide_api/src/syntax_tree.rs
index 276f8a8c8..a4e4c3dbe 100644
--- a/crates/ra_ide_api/src/syntax_tree.rs
+++ b/crates/ra_ide_api/src/syntax_tree.rs
@@ -1,8 +1,9 @@
 use ra_db::SourceDatabase;

 use crate::db::RootDatabase;

 use ra_syntax::{

-    SourceFile, SyntaxNode, TextRange, AstNode,

-    algo::{self, visit::{visitor, Visitor}}, ast::{self, AstToken}

+    SourceFile, TextRange, AstNode, SyntaxToken, SyntaxElement,

+    algo,

+    SyntaxKind::{STRING, RAW_STRING},

 };

 

 pub use ra_db::FileId;

@@ -14,11 +15,15 @@ pub(crate) fn syntax_tree(
 ) -> String {

     if let Some(text_range) = text_range {

         let file = db.parse(file_id);

-        let node = algo::find_covering_node(file.syntax(), text_range);

-

-        if let Some(tree) = syntax_tree_for_string(node, text_range) {

-            return tree;

-        }

+        let node = match algo::find_covering_element(file.syntax(), text_range) {

+            SyntaxElement::Node(node) => node,

+            SyntaxElement::Token(token) => {

+                if let Some(tree) = syntax_tree_for_string(token, text_range) {

+                    return tree;

+                }

+                token.parent()

+            }

+        };

 

         node.debug_dump()

     } else {

@@ -28,19 +33,19 @@ pub(crate) fn syntax_tree(
 

 /// Attempts parsing the selected contents of a string literal

 /// as rust syntax and returns its syntax tree

-fn syntax_tree_for_string(node: &SyntaxNode, text_range: TextRange) -> Option<String> {

+fn syntax_tree_for_string(token: SyntaxToken, text_range: TextRange) -> Option<String> {

     // When the range is inside a string

     // we'll attempt parsing it as rust syntax

     // to provide the syntax tree of the contents of the string

-    visitor()

-        .visit(|node: &ast::String| syntax_tree_for_token(node, text_range))

-        .visit(|node: &ast::RawString| syntax_tree_for_token(node, text_range))

-        .accept(node)?

+    match token.kind() {

+        STRING | RAW_STRING => syntax_tree_for_token(token, text_range),

+        _ => None,

+    }

 }

 

-fn syntax_tree_for_token<T: AstToken>(node: &T, text_range: TextRange) -> Option<String> {

+fn syntax_tree_for_token(node: SyntaxToken, text_range: TextRange) -> Option<String> {

     // Range of the full node

-    let node_range = node.syntax().range();

+    let node_range = node.range();

     let text = node.text().to_string();

 

     // We start at some point inside the node

diff --git a/crates/ra_ide_api/src/typing.rs b/crates/ra_ide_api/src/typing.rs
index 501d44dbb..4510d663d 100644
--- a/crates/ra_ide_api/src/typing.rs
+++ b/crates/ra_ide_api/src/typing.rs
@@ -1,8 +1,8 @@
 use ra_syntax::{
     AstNode, SourceFile, SyntaxKind::*,
-    SyntaxNode, TextUnit, TextRange,
-    algo::{find_node_at_offset, find_leaf_at_offset, LeafAtOffset},
-    ast::{self, AstToken},
+    TextUnit, TextRange, SyntaxToken,
+    algo::{find_node_at_offset, find_token_at_offset, TokenAtOffset},
+    ast::{self},
 };
 use ra_fmt::leading_indent;
 use ra_text_edit::{TextEdit, TextEditBuilder};
@@ -11,11 +11,11 @@ use crate::{db::RootDatabase, SourceChange, SourceFileEdit};
 
 pub(crate) fn on_enter(db: &RootDatabase, position: FilePosition) -> Option<SourceChange> {
     let file = db.parse(position.file_id);
-    let comment = find_leaf_at_offset(file.syntax(), position.offset)
+    let comment = find_token_at_offset(file.syntax(), position.offset)
         .left_biased()
         .and_then(ast::Comment::cast)?;
 
-    if let ast::CommentFlavor::Multiline = comment.flavor() {
+    if comment.flavor() == ast::CommentFlavor::Multiline {
         return None;
     }
 
@@ -41,23 +41,23 @@ pub(crate) fn on_enter(db: &RootDatabase, position: FilePosition) -> Option<Sour
     )
 }
 
-fn node_indent<'a>(file: &'a SourceFile, node: &SyntaxNode) -> Option<&'a str> {
-    let ws = match find_leaf_at_offset(file.syntax(), node.range().start()) {
-        LeafAtOffset::Between(l, r) => {
-            assert!(r == node);
+fn node_indent<'a>(file: &'a SourceFile, token: SyntaxToken) -> Option<&'a str> {
+    let ws = match find_token_at_offset(file.syntax(), token.range().start()) {
+        TokenAtOffset::Between(l, r) => {
+            assert!(r == token);
             l
         }
-        LeafAtOffset::Single(n) => {
-            assert!(n == node);
+        TokenAtOffset::Single(n) => {
+            assert!(n == token);
             return Some("");
         }
-        LeafAtOffset::None => unreachable!(),
+        TokenAtOffset::None => unreachable!(),
     };
     if ws.kind() != WHITESPACE {
         return None;
     }
-    let text = ws.leaf_text().unwrap();
-    let pos = text.as_str().rfind('\n').map(|it| it + 1).unwrap_or(0);
+    let text = ws.text();
+    let pos = text.rfind('\n').map(|it| it + 1).unwrap_or(0);
     Some(&text[pos..])
 }
 
@@ -88,7 +88,7 @@ pub(crate) fn on_dot_typed(db: &RootDatabase, position: FilePosition) -> Option<
     let file = db.parse(position.file_id);
     assert_eq!(file.syntax().text().char_at(position.offset), Some('.'));
 
-    let whitespace = find_leaf_at_offset(file.syntax(), position.offset)
+    let whitespace = find_token_at_offset(file.syntax(), position.offset)
         .left_biased()
         .and_then(ast::Whitespace::cast)?;
 
@@ -100,7 +100,7 @@ pub(crate) fn on_dot_typed(db: &RootDatabase, position: FilePosition) -> Option<
     let current_indent_len = TextUnit::of_str(current_indent);
 
     // Make sure dot is a part of call chain
-    let field_expr = whitespace.syntax().parent().and_then(ast::FieldExpr::cast)?;
+    let field_expr = ast::FieldExpr::cast(whitespace.syntax().parent())?;
     let prev_indent = leading_indent(field_expr.syntax())?;
     let target_indent = format!("    {}", prev_indent);
     let target_indent_len = TextUnit::of_str(&target_indent);