1 files changed, 262 insertions, 34 deletions

diff --git a/crates/ra_syntax/src/syntax_node.rs b/crates/ra_syntax/src/syntax_node.rs
index e5b4cdb11..a88a348ad 100644
--- a/crates/ra_syntax/src/syntax_node.rs
+++ b/crates/ra_syntax/src/syntax_node.rs
@@ -29,6 +29,9 @@ impl Types for RaTypes {
 }
 
 pub(crate) type GreenNode = rowan::GreenNode<RaTypes>;
+pub(crate) type GreenToken = rowan::GreenToken<RaTypes>;
+#[allow(unused)]
+pub(crate) type GreenElement = rowan::GreenElement<RaTypes>;
 
 /// Marker trait for CST and AST nodes
 pub trait SyntaxNodeWrapper: TransparentNewType<Repr = rowan::SyntaxNode<RaTypes>> {}
@@ -113,11 +116,13 @@ impl ToOwned for SyntaxNode {
 
 impl fmt::Debug for SyntaxNode {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        write!(fmt, "{:?}@{:?}", self.kind(), self.range())?;
-        if has_short_text(self.kind()) {
-            write!(fmt, " \"{}\"", self.text())?;
-        }
-        Ok(())
+        write!(fmt, "{:?}@{:?}", self.kind(), self.range())
+    }
+}
+
+impl fmt::Display for SyntaxNode {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Display::fmt(&self.text(), fmt)
     }
 }
 
@@ -145,14 +150,6 @@ impl SyntaxNode {
         SyntaxText::new(self)
     }
 
-    pub fn is_leaf(&self) -> bool {
-        self.0.is_leaf()
-    }
-
-    pub fn leaf_text(&self) -> Option<&SmolStr> {
-        self.0.leaf_text()
-    }
-
     pub fn parent(&self) -> Option<&SyntaxNode> {
         self.0.parent().map(SyntaxNode::from_repr)
     }
@@ -161,22 +158,50 @@ impl SyntaxNode {
         self.0.first_child().map(SyntaxNode::from_repr)
     }
 
+    pub fn first_child_or_token(&self) -> Option<SyntaxElement> {
+        self.0.first_child_or_token().map(SyntaxElement::from)
+    }
+
     pub fn last_child(&self) -> Option<&SyntaxNode> {
         self.0.last_child().map(SyntaxNode::from_repr)
     }
 
+    pub fn last_child_or_token(&self) -> Option<SyntaxElement> {
+        self.0.last_child_or_token().map(SyntaxElement::from)
+    }
+
     pub fn next_sibling(&self) -> Option<&SyntaxNode> {
         self.0.next_sibling().map(SyntaxNode::from_repr)
     }
 
+    pub fn next_sibling_or_token(&self) -> Option<SyntaxElement> {
+        self.0.next_sibling_or_token().map(SyntaxElement::from)
+    }
+
     pub fn prev_sibling(&self) -> Option<&SyntaxNode> {
         self.0.prev_sibling().map(SyntaxNode::from_repr)
     }
 
+    pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement> {
+        self.0.prev_sibling_or_token().map(SyntaxElement::from)
+    }
+
     pub fn children(&self) -> SyntaxNodeChildren {
         SyntaxNodeChildren(self.0.children())
     }
 
+    pub fn children_with_tokens(&self) -> SyntaxElementChildren {
+        SyntaxElementChildren(self.0.children_with_tokens())
+    }
+
+    pub fn first_token(&self) -> Option<SyntaxToken> {
+        self.0.first_token().map(SyntaxToken::from)
+    }
+
+    pub fn last_token(&self) -> Option<SyntaxToken> {
+        self.0.last_token().map(SyntaxToken::from)
+    }
+
     pub fn ancestors(&self) -> impl Iterator<Item = &SyntaxNode> {
         crate::algo::generate(Some(self), |&node| node.parent())
     }
@@ -188,6 +213,13 @@ impl SyntaxNode {
         })
     }
 
+    pub fn descendants_with_tokens(&self) -> impl Iterator<Item = SyntaxElement> {
+        self.preorder_with_tokens().filter_map(|event| match event {
+            WalkEvent::Enter(it) => Some(it),
+            WalkEvent::Leave(_) => None,
+        })
+    }
+
     pub fn siblings(&self, direction: Direction) -> impl Iterator<Item = &SyntaxNode> {
         crate::algo::generate(Some(self), move |&node| match direction {
             Direction::Next => node.next_sibling(),
@@ -195,6 +227,17 @@ impl SyntaxNode {
         })
     }
 
+    pub fn siblings_with_tokens(
+        &self,
+        direction: Direction,
+    ) -> impl Iterator<Item = SyntaxElement> {
+        let me: SyntaxElement = self.into();
+        crate::algo::generate(Some(me), move |el| match direction {
+            Direction::Next => el.next_sibling_or_token(),
+            Direction::Prev => el.prev_sibling_or_token(),
+        })
+    }
+
     pub fn preorder(&self) -> impl Iterator<Item = WalkEvent<&SyntaxNode>> {
         self.0.preorder().map(|event| match event {
             WalkEvent::Enter(n) => WalkEvent::Enter(SyntaxNode::from_repr(n)),
@@ -202,6 +245,13 @@ impl SyntaxNode {
         })
     }
 
+    pub fn preorder_with_tokens(&self) -> impl Iterator<Item = WalkEvent<SyntaxElement>> {
+        self.0.preorder_with_tokens().map(|event| match event {
+            WalkEvent::Enter(n) => WalkEvent::Enter(n.into()),
+            WalkEvent::Leave(n) => WalkEvent::Leave(n.into()),
+        })
+    }
+
     pub fn memory_size_of_subtree(&self) -> usize {
         self.0.memory_size_of_subtree()
     }
@@ -223,17 +273,20 @@ impl SyntaxNode {
             };
         }
 
-        for event in self.preorder() {
+        for event in self.preorder_with_tokens() {
             match event {
-                WalkEvent::Enter(node) => {
+                WalkEvent::Enter(element) => {
                     indent!();
-                    writeln!(buf, "{:?}", node).unwrap();
-                    if node.first_child().is_none() {
-                        let off = node.range().end();
-                        while err_pos < errors.len() && errors[err_pos].offset() <= off {
-                            indent!();
-                            writeln!(buf, "err: `{}`", errors[err_pos]).unwrap();
-                            err_pos += 1;
+                    match element {
+                        SyntaxElement::Node(node) => writeln!(buf, "{:?}", node).unwrap(),
+                        SyntaxElement::Token(token) => {
+                            writeln!(buf, "{:?}", token).unwrap();
+                            let off = token.range().end();
+                            while err_pos < errors.len() && errors[err_pos].offset() <= off {
+                                indent!();
+                                writeln!(buf, "err: `{}`", errors[err_pos]).unwrap();
+                                err_pos += 1;
+                            }
                         }
                     }
                     level += 1;
@@ -255,7 +308,179 @@ impl SyntaxNode {
     }
 
     pub(crate) fn replace_with(&self, replacement: GreenNode) -> GreenNode {
-        self.0.replace_self(replacement)
+        self.0.replace_with(replacement)
+    }
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SyntaxToken<'a>(pub(crate) rowan::SyntaxToken<'a, RaTypes>);
+
+//FIXME: always output text
+impl<'a> fmt::Debug for SyntaxToken<'a> {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        write!(fmt, "{:?}@{:?}", self.kind(), self.range())?;
+        if self.text().len() < 25 {
+            return write!(fmt, " {:?}", self.text());
+        }
+        let text = self.text().as_str();
+        for idx in 21..25 {
+            if text.is_char_boundary(idx) {
+                let text = format!("{} ...", &text[..idx]);
+                return write!(fmt, " {:?}", text);
+            }
+        }
+        unreachable!()
+    }
+}
+
+impl<'a> fmt::Display for SyntaxToken<'a> {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Display::fmt(self.text(), fmt)
+    }
+}
+
+impl<'a> From<rowan::SyntaxToken<'a, RaTypes>> for SyntaxToken<'a> {
+    fn from(t: rowan::SyntaxToken<'a, RaTypes>) -> Self {
+        SyntaxToken(t)
+    }
+}
+
+impl<'a> SyntaxToken<'a> {
+    pub fn kind(&self) -> SyntaxKind {
+        self.0.kind()
+    }
+
+    pub fn text(&self) -> &'a SmolStr {
+        self.0.text()
+    }
+
+    pub fn range(&self) -> TextRange {
+        self.0.range()
+    }
+
+    pub fn parent(&self) -> &'a SyntaxNode {
+        SyntaxNode::from_repr(self.0.parent())
+    }
+
+    pub fn next_sibling_or_token(&self) -> Option<SyntaxElement<'a>> {
+        self.0.next_sibling_or_token().map(SyntaxElement::from)
+    }
+
+    pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement<'a>> {
+        self.0.prev_sibling_or_token().map(SyntaxElement::from)
+    }
+
+    pub fn siblings_with_tokens(
+        &self,
+        direction: Direction,
+    ) -> impl Iterator<Item = SyntaxElement<'a>> {
+        let me: SyntaxElement = (*self).into();
+        crate::algo::generate(Some(me), move |el| match direction {
+            Direction::Next => el.next_sibling_or_token(),
+            Direction::Prev => el.prev_sibling_or_token(),
+        })
+    }
+
+    pub fn next_token(&self) -> Option<SyntaxToken<'a>> {
+        self.0.next_token().map(SyntaxToken::from)
+    }
+
+    pub fn prev_token(&self) -> Option<SyntaxToken<'a>> {
+        self.0.prev_token().map(SyntaxToken::from)
+    }
+
+    pub(crate) fn replace_with(&self, new_token: GreenToken) -> GreenNode {
+        self.0.replace_with(new_token)
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
+pub enum SyntaxElement<'a> {
+    Node(&'a SyntaxNode),
+    Token(SyntaxToken<'a>),
+}
+
+impl<'a> fmt::Display for SyntaxElement<'a> {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            SyntaxElement::Node(it) => fmt::Display::fmt(it, fmt),
+            SyntaxElement::Token(it) => fmt::Display::fmt(it, fmt),
+        }
+    }
+}
+
+impl<'a> SyntaxElement<'a> {
+    pub fn kind(&self) -> SyntaxKind {
+        match self {
+            SyntaxElement::Node(it) => it.kind(),
+            SyntaxElement::Token(it) => it.kind(),
+        }
+    }
+
+    pub fn as_node(&self) -> Option<&'a SyntaxNode> {
+        match self {
+            SyntaxElement::Node(node) => Some(*node),
+            SyntaxElement::Token(_) => None,
+        }
+    }
+
+    pub fn as_token(&self) -> Option<SyntaxToken<'a>> {
+        match self {
+            SyntaxElement::Node(_) => None,
+            SyntaxElement::Token(token) => Some(*token),
+        }
+    }
+
+    pub fn next_sibling_or_token(&self) -> Option<SyntaxElement<'a>> {
+        match self {
+            SyntaxElement::Node(it) => it.next_sibling_or_token(),
+            SyntaxElement::Token(it) => it.next_sibling_or_token(),
+        }
+    }
+
+    pub fn prev_sibling_or_token(&self) -> Option<SyntaxElement<'a>> {
+        match self {
+            SyntaxElement::Node(it) => it.prev_sibling_or_token(),
+            SyntaxElement::Token(it) => it.prev_sibling_or_token(),
+        }
+    }
+
+    pub fn ancestors(&self) -> impl Iterator<Item = &'a SyntaxNode> {
+        match self {
+            SyntaxElement::Node(it) => it,
+            SyntaxElement::Token(it) => it.parent(),
+        }
+        .ancestors()
+    }
+}
+
+impl<'a> From<rowan::SyntaxElement<'a, RaTypes>> for SyntaxElement<'a> {
+    fn from(el: rowan::SyntaxElement<'a, RaTypes>) -> Self {
+        match el {
+            rowan::SyntaxElement::Node(n) => SyntaxElement::Node(SyntaxNode::from_repr(n)),
+            rowan::SyntaxElement::Token(t) => SyntaxElement::Token(t.into()),
+        }
+    }
+}
+
+impl<'a> From<&'a SyntaxNode> for SyntaxElement<'a> {
+    fn from(node: &'a SyntaxNode) -> SyntaxElement<'a> {
+        SyntaxElement::Node(node)
+    }
+}
+
+impl<'a> From<SyntaxToken<'a>> for SyntaxElement<'a> {
+    fn from(token: SyntaxToken<'a>) -> SyntaxElement<'a> {
+        SyntaxElement::Token(token)
+    }
+}
+
+impl<'a> SyntaxElement<'a> {
+    pub fn range(&self) -> TextRange {
+        match self {
+            SyntaxElement::Node(it) => it.range(),
+            SyntaxElement::Token(it) => it.range(),
+        }
     }
 }
 
@@ -270,11 +495,14 @@ impl<'a> Iterator for SyntaxNodeChildren<'a> {
     }
 }
 
-fn has_short_text(kind: SyntaxKind) -> bool {
-    use crate::SyntaxKind::*;
-    match kind {
-        IDENT | LIFETIME | INT_NUMBER | FLOAT_NUMBER => true,
-        _ => false,
+#[derive(Debug)]
+pub struct SyntaxElementChildren<'a>(rowan::SyntaxElementChildren<'a, RaTypes>);
+
+impl<'a> Iterator for SyntaxElementChildren<'a> {
+    type Item = SyntaxElement<'a>;
+
+    fn next(&mut self) -> Option<SyntaxElement<'a>> {
+        self.0.next().map(SyntaxElement::from)
     }
 }
 
@@ -304,16 +532,16 @@ impl SyntaxTreeBuilder {
         node
     }
 
-    pub fn leaf(&mut self, kind: SyntaxKind, text: SmolStr) {
-        self.inner.leaf(kind, text)
+    pub fn token(&mut self, kind: SyntaxKind, text: SmolStr) {
+        self.inner.token(kind, text)
     }
 
-    pub fn start_branch(&mut self, kind: SyntaxKind) {
-        self.inner.start_internal(kind)
+    pub fn start_node(&mut self, kind: SyntaxKind) {
+        self.inner.start_node(kind)
     }
 
-    pub fn finish_branch(&mut self) {
-        self.inner.finish_internal()
+    pub fn finish_node(&mut self) {
+        self.inner.finish_node()
     }
 
     pub fn error(&mut self, error: ParseError, text_pos: TextUnit) {