Merge #1312

1312: Introduce TokenBuffer r=matklad a=edwin0cheng

As discussed in Zulip, this PR Introduce `TokenBuffer` , a safe version of  `syn` crate `TokenBuffer` which support cursor based traversal of `tt::TokenTree`. This is the basis of incoming refactoring of `TokenSource` iterator based API.

This PR do the following things:

* Add TokenBuffer in `ra_tt` crate.
* Try to use this new API to refactor the `SubtreeSource` to prove it usage.




Co-authored-by: Edwin Cheng <[email protected]>

1 files changed, 4 insertions, 2 deletions

diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index f07107414..709b87a38 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -2,6 +2,7 @@ use crate::subtree_source::SubtreeTokenSource;
 
 use ra_parser::{TokenSource, TreeSink};
 use ra_syntax::{SyntaxKind};
+use tt::buffer::TokenBuffer;
 
 struct OffsetTokenSink {
     token_pos: usize,
@@ -69,7 +70,8 @@ impl<'a> Parser<'a> {
     where
         F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
     {
-        let mut src = SubtreeTokenSource::new(&self.subtree.token_trees[*self.cur_pos..]);
+        let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
+        let mut src = SubtreeTokenSource::new(&buffer);
         let mut sink = OffsetTokenSink { token_pos: 0, error: false };
 
         f(&src, &mut sink);
@@ -85,7 +87,7 @@ impl<'a> Parser<'a> {
         let res = src.bump_n(parsed_token);
         *self.cur_pos += res.len();
 
-        let res: Vec<_> = res.into_iter().cloned().collect();
+        let res: Vec<_> = res.into_iter().collect();
 
         match res.len() {
             0 => None,