Merge #1368

1368: Store referece instead of full token tree in tokenbuffer r=matklad a=edwin0cheng

This PR try to minimize the memory allocation in converting `SyntaxNode` to `TokenTree` by using reference isnteead of full token tree in `TokenBuffer`. 

Note that the final goal is replace `TokenTree` with TokenBuffer such that there is no conversion between them.

Co-authored-by: Edwin Cheng <[email protected]>

2 files changed, 6 insertions, 4 deletions

diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index 1f12e42ef..4a6f6aa45 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -10,7 +10,7 @@ struct OffsetTokenSink<'a> {
 }
 
 impl<'a> OffsetTokenSink<'a> {
-    pub fn collect(&self, begin: Cursor<'a>) -> Vec<tt::TokenTree> {
+    pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> {
         if !self.cursor.is_root() {
             return vec![];
         }
@@ -114,7 +114,7 @@ impl<'a> Parser<'a> {
             1 => Some(res[0].clone()),
             _ => Some(tt::TokenTree::Subtree(tt::Subtree {
                 delimiter: tt::Delimiter::None,
-                token_trees: res,
+                token_trees: res.into_iter().cloned().collect(),
             })),
         }
     }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index dce82f33d..0edb6f9a2 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -49,7 +49,8 @@ fn token_tree_to_syntax_node<F>(tt: &tt::Subtree, f: F) -> Result<TreeArc<Syntax
 where
     F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink),
 {
-    let buffer = TokenBuffer::new(&[tt.clone().into()]);
+    let tokens = [tt.clone().into()];
+    let buffer = TokenBuffer::new(&tokens);
     let mut token_source = SubtreeTokenSource::new(&buffer);
     let mut tree_sink = TtTreeSink::new(buffer.begin());
     f(&mut token_source, &mut tree_sink);
@@ -385,7 +386,8 @@ mod tests {
             "#,
         );
         let expansion = expand(&rules, "literals!(foo);");
-        let buffer = tt::buffer::TokenBuffer::new(&[expansion.clone().into()]);
+        let tts = &[expansion.clone().into()];
+        let buffer = tt::buffer::TokenBuffer::new(tts);
         let mut tt_src = SubtreeTokenSource::new(&buffer);
         let mut tokens = vec![];
         while tt_src.current().kind != EOF {