Merge #1328

1328: Change TokenSource to iteration based r=matklad a=edwin0cheng

This PR change the `TokenSource` trait from random access to be an iteration based trait:
```rust

/// `TokenSource` abstracts the source of the tokens parser operates one.
///
/// Hopefully this will allow us to treat text and token trees in the same way!
pub trait TokenSource {
    fn current(&self) -> Token;

    /// Lookahead n token
    fn lookahead_nth(&self, n: usize) -> Token;

    /// bump cursor to next token
    fn bump(&mut self);

    /// Is the current token a specified keyword?
    fn is_keyword(&self, kw: &str) -> bool;
}

/// `TokenCursor` abstracts the cursor of `TokenSource` operates one.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub struct Token {
    /// What is the current token?
    pub kind: SyntaxKind,

    /// Is the current token joined to the next one (`> >` vs `>>`).
    pub is_jointed_to_next: bool,
}
```

Note that the refactoring based on this new trait will be separated to incoming PRs

Co-authored-by: Edwin Cheng <[email protected]>

3 files changed, 59 insertions, 67 deletions

diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index 709b87a38..9cc989b23 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -68,13 +68,13 @@ impl<'a> Parser<'a> {
 
     fn parse<F>(self, f: F) -> Option<tt::TokenTree>
     where
-        F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
+        F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
     {
         let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
         let mut src = SubtreeTokenSource::new(&buffer);
         let mut sink = OffsetTokenSink { token_pos: 0, error: false };
 
-        f(&src, &mut sink);
+        f(&mut src, &mut sink);
 
         let r = self.finish(sink.token_pos, &mut src);
         if sink.error {
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 972af4a7c..c4f79f38a 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -1,6 +1,7 @@
-use ra_parser::{TokenSource};
+use ra_parser::{TokenSource, Token};
 use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use std::cell::{RefCell, Cell};
+use std::sync::Arc;
 use tt::buffer::{TokenBuffer, Cursor};
 
 pub(crate) trait Querier {
@@ -65,7 +66,7 @@ impl<'a> SubtreeWalk<'a> {
         return cached[pos].clone();
     }
 
-    fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> {
+    fn collect_token_trees(&self, n: usize) -> Vec<tt::TokenTree> {
         let mut res = vec![];
 
         let mut pos = 0;
@@ -117,43 +118,59 @@ impl<'a> Querier for SubtreeWalk<'a> {
 }
 
 pub(crate) struct SubtreeTokenSource<'a> {
-    walker: SubtreeWalk<'a>,
+    walker: Arc<SubtreeWalk<'a>>,
+    curr: (Token, usize),
 }
 
 impl<'a> SubtreeTokenSource<'a> {
     pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
-        SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) }
+        let mut res = SubtreeTokenSource {
+            walker: Arc::new(SubtreeWalk::new(buffer.begin())),
+            curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
+        };
+        res.curr = (res.mk_token(0), 0);
+        res
     }
 
-    pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a>
-    where
-        'a: 'b,
-    {
-        &self.walker
+    pub fn querier(&self) -> Arc<SubtreeWalk<'a>> {
+        self.walker.clone()
     }
 
     pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> {
         let res = self.walker.collect_token_trees(parsed_tokens);
         res
     }
+
+    fn mk_token(&self, pos: usize) -> Token {
+        match self.walker.get(pos) {
+            Some(tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
+            None => Token { kind: EOF, is_jointed_to_next: false },
+        }
+    }
 }
 
 impl<'a> TokenSource for SubtreeTokenSource<'a> {
-    fn token_kind(&self, pos: usize) -> SyntaxKind {
-        if let Some(tok) = self.walker.get(pos) {
-            tok.kind
-        } else {
-            SyntaxKind::EOF
-        }
+    fn current(&self) -> Token {
+        self.curr.0
     }
-    fn is_token_joint_to_next(&self, pos: usize) -> bool {
-        match self.walker.get(pos) {
-            Some(t) => t.is_joint_to_next,
-            _ => false,
+
+    /// Lookahead n token
+    fn lookahead_nth(&self, n: usize) -> Token {
+        self.mk_token(self.curr.1 + n)
+    }
+
+    /// bump cursor to next token
+    fn bump(&mut self) {
+        if self.current().kind == EOF {
+            return;
         }
+
+        self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1)
     }
-    fn is_keyword(&self, pos: usize, kw: &str) -> bool {
-        match self.walker.get(pos) {
+
+    /// Is the current token a specified keyword?
+    fn is_keyword(&self, kw: &str) -> bool {
+        match self.walker.get(self.curr.1) {
             Some(t) => t.text == *kw,
             _ => false,
         }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 0a75305b4..0a7e50c4e 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -45,17 +45,25 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
 //
 //
 
-/// Parses the token tree (result of macro expansion) to an expression
-pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
+fn token_tree_to_syntax_node<F>(tt: &tt::Subtree, f: F) -> Result<TreeArc<SyntaxNode>, ExpandError>
+where
+    F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink),
+{
     let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_expr(&token_source, &mut tree_sink);
+    let mut token_source = SubtreeTokenSource::new(&buffer);
+    let querier = token_source.querier();
+    let mut tree_sink = TtTreeSink::new(querier.as_ref());
+    f(&mut token_source, &mut tree_sink);
     if tree_sink.roots.len() != 1 {
         return Err(ExpandError::ConversionError);
     }
 
-    let syntax = tree_sink.inner.finish();
+    Ok(tree_sink.inner.finish())
+}
+
+/// Parses the token tree (result of macro expansion) to an expression
+pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_expr)?;
     ast::Expr::cast(&syntax)
         .map(|m| m.to_owned())
         .ok_or_else(|| crate::ExpandError::ConversionError)
@@ -63,28 +71,13 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, Expand
 
 /// Parses the token tree (result of macro expansion) to a Pattern
 pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_pat(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-
-    let syntax = tree_sink.inner.finish();
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_pat)?;
     ast::Pat::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
 /// Parses the token tree (result of macro expansion) to a Type
 pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_ty(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-    let syntax = tree_sink.inner.finish();
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_ty)?;
     ast::TypeRef::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
@@ -92,14 +85,7 @@ pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, Expan
 pub fn token_tree_to_macro_stmts(
     tt: &tt::Subtree,
 ) -> Result<TreeArc<ast::MacroStmts>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_macro_stmts(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-    let syntax = tree_sink.inner.finish();
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_stmts)?;
     ast::MacroStmts::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
@@ -107,24 +93,13 @@ pub fn token_tree_to_macro_stmts(
 pub fn token_tree_to_macro_items(
     tt: &tt::Subtree,
 ) -> Result<TreeArc<ast::MacroItems>, ExpandError> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse_macro_items(&token_source, &mut tree_sink);
-    if tree_sink.roots.len() != 1 {
-        return Err(ExpandError::ConversionError);
-    }
-    let syntax = tree_sink.inner.finish();
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse_macro_items)?;
     ast::MacroItems::cast(&syntax).map(|m| m.to_owned()).ok_or_else(|| ExpandError::ConversionError)
 }
 
 /// Parses the token tree (result of macro expansion) as a sequence of items
 pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> {
-    let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
-    let token_source = SubtreeTokenSource::new(&buffer);
-    let mut tree_sink = TtTreeSink::new(token_source.querier());
-    ra_parser::parse(&token_source, &mut tree_sink);
-    let syntax = tree_sink.inner.finish();
+    let syntax = token_tree_to_syntax_node(tt, ra_parser::parse).unwrap();
     ast::SourceFile::cast(&syntax).unwrap().to_owned()
 }