aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbors[bot] <bors[bot]@users.noreply.github.com>2019-06-02 18:09:49 +0100
committerbors[bot] <bors[bot]@users.noreply.github.com>2019-06-02 18:09:49 +0100
commitae8fd982c05c4d825952c78d1b1d8a5cfba94e66 (patch)
treec9cdc209f904cade7bc2deac2489239da34f7126
parentb40c6de8a6887e6c184fca5c9188d26ee402df23 (diff)
parent824f413d75b68e950726dc17c4e7cc3d3f241eda (diff)
Merge #1368
1368: Store referece instead of full token tree in tokenbuffer r=matklad a=edwin0cheng This PR try to minimize the memory allocation in converting `SyntaxNode` to `TokenTree` by using reference isnteead of full token tree in `TokenBuffer`. Note that the final goal is replace `TokenTree` with TokenBuffer such that there is no conversion between them. Co-authored-by: Edwin Cheng <[email protected]>
-rw-r--r--crates/ra_mbe/src/subtree_parser.rs4
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs6
-rw-r--r--crates/ra_tt/src/buffer.rs44
3 files changed, 28 insertions, 26 deletions
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs
index 1f12e42ef..4a6f6aa45 100644
--- a/crates/ra_mbe/src/subtree_parser.rs
+++ b/crates/ra_mbe/src/subtree_parser.rs
@@ -10,7 +10,7 @@ struct OffsetTokenSink<'a> {
10} 10}
11 11
12impl<'a> OffsetTokenSink<'a> { 12impl<'a> OffsetTokenSink<'a> {
13 pub fn collect(&self, begin: Cursor<'a>) -> Vec<tt::TokenTree> { 13 pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> {
14 if !self.cursor.is_root() { 14 if !self.cursor.is_root() {
15 return vec![]; 15 return vec![];
16 } 16 }
@@ -114,7 +114,7 @@ impl<'a> Parser<'a> {
114 1 => Some(res[0].clone()), 114 1 => Some(res[0].clone()),
115 _ => Some(tt::TokenTree::Subtree(tt::Subtree { 115 _ => Some(tt::TokenTree::Subtree(tt::Subtree {
116 delimiter: tt::Delimiter::None, 116 delimiter: tt::Delimiter::None,
117 token_trees: res, 117 token_trees: res.into_iter().cloned().collect(),
118 })), 118 })),
119 } 119 }
120 } 120 }
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index dce82f33d..0edb6f9a2 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -49,7 +49,8 @@ fn token_tree_to_syntax_node<F>(tt: &tt::Subtree, f: F) -> Result<TreeArc<Syntax
49where 49where
50 F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink), 50 F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink),
51{ 51{
52 let buffer = TokenBuffer::new(&[tt.clone().into()]); 52 let tokens = [tt.clone().into()];
53 let buffer = TokenBuffer::new(&tokens);
53 let mut token_source = SubtreeTokenSource::new(&buffer); 54 let mut token_source = SubtreeTokenSource::new(&buffer);
54 let mut tree_sink = TtTreeSink::new(buffer.begin()); 55 let mut tree_sink = TtTreeSink::new(buffer.begin());
55 f(&mut token_source, &mut tree_sink); 56 f(&mut token_source, &mut tree_sink);
@@ -385,7 +386,8 @@ mod tests {
385 "#, 386 "#,
386 ); 387 );
387 let expansion = expand(&rules, "literals!(foo);"); 388 let expansion = expand(&rules, "literals!(foo);");
388 let buffer = tt::buffer::TokenBuffer::new(&[expansion.clone().into()]); 389 let tts = &[expansion.clone().into()];
390 let buffer = tt::buffer::TokenBuffer::new(tts);
389 let mut tt_src = SubtreeTokenSource::new(&buffer); 391 let mut tt_src = SubtreeTokenSource::new(&buffer);
390 let mut tokens = vec![]; 392 let mut tokens = vec![];
391 while tt_src.current().kind != EOF { 393 while tt_src.current().kind != EOF {
diff --git a/crates/ra_tt/src/buffer.rs b/crates/ra_tt/src/buffer.rs
index 940f2b807..995c9f90b 100644
--- a/crates/ra_tt/src/buffer.rs
+++ b/crates/ra_tt/src/buffer.rs
@@ -1,4 +1,4 @@
1use crate::{TokenTree, Subtree, Leaf}; 1use crate::{TokenTree, Subtree};
2 2
3#[derive(Copy, Clone, Debug, Eq, PartialEq)] 3#[derive(Copy, Clone, Debug, Eq, PartialEq)]
4struct EntryId(usize); 4struct EntryId(usize);
@@ -9,10 +9,10 @@ struct EntryPtr(EntryId, usize);
9/// Internal type which is used instead of `TokenTree` to represent a token tree 9/// Internal type which is used instead of `TokenTree` to represent a token tree
10/// within a `TokenBuffer`. 10/// within a `TokenBuffer`.
11#[derive(Debug)] 11#[derive(Debug)]
12enum Entry { 12enum Entry<'t> {
13 // Mimicking types from proc-macro. 13 // Mimicking types from proc-macro.
14 Subtree(Subtree, EntryId), 14 Subtree(&'t TokenTree, EntryId),
15 Leaf(Leaf), 15 Leaf(&'t TokenTree),
16 // End entries contain a pointer to the entry from the containing 16 // End entries contain a pointer to the entry from the containing
17 // token tree, or None if this is the outermost level. 17 // token tree, or None if this is the outermost level.
18 End(Option<EntryPtr>), 18 End(Option<EntryPtr>),
@@ -21,12 +21,12 @@ enum Entry {
21/// A token tree buffer 21/// A token tree buffer
22/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41) 22/// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41)
23#[derive(Debug)] 23#[derive(Debug)]
24pub struct TokenBuffer { 24pub struct TokenBuffer<'t> {
25 buffers: Vec<Box<[Entry]>>, 25 buffers: Vec<Box<[Entry<'t>]>>,
26} 26}
27 27
28impl TokenBuffer { 28impl<'t> TokenBuffer<'t> {
29 pub fn new(tokens: &[TokenTree]) -> TokenBuffer { 29 pub fn new(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
30 let mut buffers = vec![]; 30 let mut buffers = vec![];
31 31
32 let idx = TokenBuffer::new_inner(tokens, &mut buffers, None); 32 let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
@@ -36,21 +36,21 @@ impl TokenBuffer {
36 } 36 }
37 37
38 fn new_inner( 38 fn new_inner(
39 tokens: &[TokenTree], 39 tokens: &'t [TokenTree],
40 buffers: &mut Vec<Box<[Entry]>>, 40 buffers: &mut Vec<Box<[Entry<'t>]>>,
41 next: Option<EntryPtr>, 41 next: Option<EntryPtr>,
42 ) -> usize { 42 ) -> usize {
43 let mut entries = vec![]; 43 let mut entries = vec![];
44 let mut children = vec![]; 44 let mut children = vec![];
45 45
46 for (idx, tt) in tokens.iter().cloned().enumerate() { 46 for (idx, tt) in tokens.iter().enumerate() {
47 match tt { 47 match tt {
48 TokenTree::Leaf(leaf) => { 48 TokenTree::Leaf(_) => {
49 entries.push(Entry::Leaf(leaf)); 49 entries.push(Entry::Leaf(tt));
50 } 50 }
51 TokenTree::Subtree(subtree) => { 51 TokenTree::Subtree(subtree) => {
52 entries.push(Entry::End(None)); 52 entries.push(Entry::End(None));
53 children.push((idx, subtree)); 53 children.push((idx, (subtree, tt)));
54 } 54 }
55 } 55 }
56 } 56 }
@@ -59,13 +59,13 @@ impl TokenBuffer {
59 let res = buffers.len(); 59 let res = buffers.len();
60 buffers.push(entries.into_boxed_slice()); 60 buffers.push(entries.into_boxed_slice());
61 61
62 for (child_idx, subtree) in children { 62 for (child_idx, (subtree, tt)) in children {
63 let idx = TokenBuffer::new_inner( 63 let idx = TokenBuffer::new_inner(
64 &subtree.token_trees, 64 &subtree.token_trees,
65 buffers, 65 buffers,
66 Some(EntryPtr(EntryId(res), child_idx + 1)), 66 Some(EntryPtr(EntryId(res), child_idx + 1)),
67 ); 67 );
68 buffers[res].as_mut()[child_idx] = Entry::Subtree(subtree, EntryId(idx)); 68 buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, EntryId(idx));
69 } 69 }
70 70
71 res 71 res
@@ -86,7 +86,7 @@ impl TokenBuffer {
86/// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125 86/// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125
87#[derive(Copy, Clone, Debug)] 87#[derive(Copy, Clone, Debug)]
88pub struct Cursor<'a> { 88pub struct Cursor<'a> {
89 buffer: &'a TokenBuffer, 89 buffer: &'a TokenBuffer<'a>,
90 ptr: EntryPtr, 90 ptr: EntryPtr,
91} 91}
92 92
@@ -113,7 +113,7 @@ impl<'a> Cursor<'a> {
113 match self.entry() { 113 match self.entry() {
114 Some(Entry::End(Some(ptr))) => { 114 Some(Entry::End(Some(ptr))) => {
115 let idx = ptr.1; 115 let idx = ptr.1;
116 if let Some(Entry::Subtree(subtree, _)) = 116 if let Some(Entry::Subtree(TokenTree::Subtree(subtree), _)) =
117 self.buffer.entry(&EntryPtr(ptr.0, idx - 1)) 117 self.buffer.entry(&EntryPtr(ptr.0, idx - 1))
118 { 118 {
119 return Some(subtree); 119 return Some(subtree);
@@ -125,7 +125,7 @@ impl<'a> Cursor<'a> {
125 } 125 }
126 } 126 }
127 127
128 fn entry(self) -> Option<(&'a Entry)> { 128 fn entry(self) -> Option<(&'a Entry<'a>)> {
129 self.buffer.entry(&self.ptr) 129 self.buffer.entry(&self.ptr)
130 } 130 }
131 131
@@ -141,10 +141,10 @@ impl<'a> Cursor<'a> {
141 } 141 }
142 142
143 /// If the cursor is pointing at a `TokenTree`, returns it 143 /// If the cursor is pointing at a `TokenTree`, returns it
144 pub fn token_tree(self) -> Option<(TokenTree)> { 144 pub fn token_tree(self) -> Option<&'a TokenTree> {
145 match self.entry() { 145 match self.entry() {
146 Some(Entry::Leaf(leaf)) => Some(leaf.clone().into()), 146 Some(Entry::Leaf(tt)) => Some(tt),
147 Some(Entry::Subtree(subtree, _)) => Some(subtree.clone().into()), 147 Some(Entry::Subtree(tt, _)) => Some(tt),
148 Some(Entry::End(_)) => None, 148 Some(Entry::End(_)) => None,
149 None => None, 149 None => None,
150 } 150 }