diff options
author | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-06-02 18:09:49 +0100 |
---|---|---|
committer | bors[bot] <bors[bot]@users.noreply.github.com> | 2019-06-02 18:09:49 +0100 |
commit | ae8fd982c05c4d825952c78d1b1d8a5cfba94e66 (patch) | |
tree | c9cdc209f904cade7bc2deac2489239da34f7126 /crates | |
parent | b40c6de8a6887e6c184fca5c9188d26ee402df23 (diff) | |
parent | 824f413d75b68e950726dc17c4e7cc3d3f241eda (diff) |
Merge #1368
1368: Store referece instead of full token tree in tokenbuffer r=matklad a=edwin0cheng
This PR try to minimize the memory allocation in converting `SyntaxNode` to `TokenTree` by using reference isnteead of full token tree in `TokenBuffer`.
Note that the final goal is replace `TokenTree` with TokenBuffer such that there is no conversion between them.
Co-authored-by: Edwin Cheng <[email protected]>
Diffstat (limited to 'crates')
-rw-r--r-- | crates/ra_mbe/src/subtree_parser.rs | 4 | ||||
-rw-r--r-- | crates/ra_mbe/src/syntax_bridge.rs | 6 | ||||
-rw-r--r-- | crates/ra_tt/src/buffer.rs | 44 |
3 files changed, 28 insertions, 26 deletions
diff --git a/crates/ra_mbe/src/subtree_parser.rs b/crates/ra_mbe/src/subtree_parser.rs index 1f12e42ef..4a6f6aa45 100644 --- a/crates/ra_mbe/src/subtree_parser.rs +++ b/crates/ra_mbe/src/subtree_parser.rs | |||
@@ -10,7 +10,7 @@ struct OffsetTokenSink<'a> { | |||
10 | } | 10 | } |
11 | 11 | ||
12 | impl<'a> OffsetTokenSink<'a> { | 12 | impl<'a> OffsetTokenSink<'a> { |
13 | pub fn collect(&self, begin: Cursor<'a>) -> Vec<tt::TokenTree> { | 13 | pub fn collect(&self, begin: Cursor<'a>) -> Vec<&'a tt::TokenTree> { |
14 | if !self.cursor.is_root() { | 14 | if !self.cursor.is_root() { |
15 | return vec![]; | 15 | return vec![]; |
16 | } | 16 | } |
@@ -114,7 +114,7 @@ impl<'a> Parser<'a> { | |||
114 | 1 => Some(res[0].clone()), | 114 | 1 => Some(res[0].clone()), |
115 | _ => Some(tt::TokenTree::Subtree(tt::Subtree { | 115 | _ => Some(tt::TokenTree::Subtree(tt::Subtree { |
116 | delimiter: tt::Delimiter::None, | 116 | delimiter: tt::Delimiter::None, |
117 | token_trees: res, | 117 | token_trees: res.into_iter().cloned().collect(), |
118 | })), | 118 | })), |
119 | } | 119 | } |
120 | } | 120 | } |
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs index dce82f33d..0edb6f9a2 100644 --- a/crates/ra_mbe/src/syntax_bridge.rs +++ b/crates/ra_mbe/src/syntax_bridge.rs | |||
@@ -49,7 +49,8 @@ fn token_tree_to_syntax_node<F>(tt: &tt::Subtree, f: F) -> Result<TreeArc<Syntax | |||
49 | where | 49 | where |
50 | F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink), | 50 | F: Fn(&mut ra_parser::TokenSource, &mut ra_parser::TreeSink), |
51 | { | 51 | { |
52 | let buffer = TokenBuffer::new(&[tt.clone().into()]); | 52 | let tokens = [tt.clone().into()]; |
53 | let buffer = TokenBuffer::new(&tokens); | ||
53 | let mut token_source = SubtreeTokenSource::new(&buffer); | 54 | let mut token_source = SubtreeTokenSource::new(&buffer); |
54 | let mut tree_sink = TtTreeSink::new(buffer.begin()); | 55 | let mut tree_sink = TtTreeSink::new(buffer.begin()); |
55 | f(&mut token_source, &mut tree_sink); | 56 | f(&mut token_source, &mut tree_sink); |
@@ -385,7 +386,8 @@ mod tests { | |||
385 | "#, | 386 | "#, |
386 | ); | 387 | ); |
387 | let expansion = expand(&rules, "literals!(foo);"); | 388 | let expansion = expand(&rules, "literals!(foo);"); |
388 | let buffer = tt::buffer::TokenBuffer::new(&[expansion.clone().into()]); | 389 | let tts = &[expansion.clone().into()]; |
390 | let buffer = tt::buffer::TokenBuffer::new(tts); | ||
389 | let mut tt_src = SubtreeTokenSource::new(&buffer); | 391 | let mut tt_src = SubtreeTokenSource::new(&buffer); |
390 | let mut tokens = vec![]; | 392 | let mut tokens = vec![]; |
391 | while tt_src.current().kind != EOF { | 393 | while tt_src.current().kind != EOF { |
diff --git a/crates/ra_tt/src/buffer.rs b/crates/ra_tt/src/buffer.rs index 940f2b807..995c9f90b 100644 --- a/crates/ra_tt/src/buffer.rs +++ b/crates/ra_tt/src/buffer.rs | |||
@@ -1,4 +1,4 @@ | |||
1 | use crate::{TokenTree, Subtree, Leaf}; | 1 | use crate::{TokenTree, Subtree}; |
2 | 2 | ||
3 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] | 3 | #[derive(Copy, Clone, Debug, Eq, PartialEq)] |
4 | struct EntryId(usize); | 4 | struct EntryId(usize); |
@@ -9,10 +9,10 @@ struct EntryPtr(EntryId, usize); | |||
9 | /// Internal type which is used instead of `TokenTree` to represent a token tree | 9 | /// Internal type which is used instead of `TokenTree` to represent a token tree |
10 | /// within a `TokenBuffer`. | 10 | /// within a `TokenBuffer`. |
11 | #[derive(Debug)] | 11 | #[derive(Debug)] |
12 | enum Entry { | 12 | enum Entry<'t> { |
13 | // Mimicking types from proc-macro. | 13 | // Mimicking types from proc-macro. |
14 | Subtree(Subtree, EntryId), | 14 | Subtree(&'t TokenTree, EntryId), |
15 | Leaf(Leaf), | 15 | Leaf(&'t TokenTree), |
16 | // End entries contain a pointer to the entry from the containing | 16 | // End entries contain a pointer to the entry from the containing |
17 | // token tree, or None if this is the outermost level. | 17 | // token tree, or None if this is the outermost level. |
18 | End(Option<EntryPtr>), | 18 | End(Option<EntryPtr>), |
@@ -21,12 +21,12 @@ enum Entry { | |||
21 | /// A token tree buffer | 21 | /// A token tree buffer |
22 | /// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41) | 22 | /// The safe version of `syn` [`TokenBuffer`](https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L41) |
23 | #[derive(Debug)] | 23 | #[derive(Debug)] |
24 | pub struct TokenBuffer { | 24 | pub struct TokenBuffer<'t> { |
25 | buffers: Vec<Box<[Entry]>>, | 25 | buffers: Vec<Box<[Entry<'t>]>>, |
26 | } | 26 | } |
27 | 27 | ||
28 | impl TokenBuffer { | 28 | impl<'t> TokenBuffer<'t> { |
29 | pub fn new(tokens: &[TokenTree]) -> TokenBuffer { | 29 | pub fn new(tokens: &'t [TokenTree]) -> TokenBuffer<'t> { |
30 | let mut buffers = vec![]; | 30 | let mut buffers = vec![]; |
31 | 31 | ||
32 | let idx = TokenBuffer::new_inner(tokens, &mut buffers, None); | 32 | let idx = TokenBuffer::new_inner(tokens, &mut buffers, None); |
@@ -36,21 +36,21 @@ impl TokenBuffer { | |||
36 | } | 36 | } |
37 | 37 | ||
38 | fn new_inner( | 38 | fn new_inner( |
39 | tokens: &[TokenTree], | 39 | tokens: &'t [TokenTree], |
40 | buffers: &mut Vec<Box<[Entry]>>, | 40 | buffers: &mut Vec<Box<[Entry<'t>]>>, |
41 | next: Option<EntryPtr>, | 41 | next: Option<EntryPtr>, |
42 | ) -> usize { | 42 | ) -> usize { |
43 | let mut entries = vec![]; | 43 | let mut entries = vec![]; |
44 | let mut children = vec![]; | 44 | let mut children = vec![]; |
45 | 45 | ||
46 | for (idx, tt) in tokens.iter().cloned().enumerate() { | 46 | for (idx, tt) in tokens.iter().enumerate() { |
47 | match tt { | 47 | match tt { |
48 | TokenTree::Leaf(leaf) => { | 48 | TokenTree::Leaf(_) => { |
49 | entries.push(Entry::Leaf(leaf)); | 49 | entries.push(Entry::Leaf(tt)); |
50 | } | 50 | } |
51 | TokenTree::Subtree(subtree) => { | 51 | TokenTree::Subtree(subtree) => { |
52 | entries.push(Entry::End(None)); | 52 | entries.push(Entry::End(None)); |
53 | children.push((idx, subtree)); | 53 | children.push((idx, (subtree, tt))); |
54 | } | 54 | } |
55 | } | 55 | } |
56 | } | 56 | } |
@@ -59,13 +59,13 @@ impl TokenBuffer { | |||
59 | let res = buffers.len(); | 59 | let res = buffers.len(); |
60 | buffers.push(entries.into_boxed_slice()); | 60 | buffers.push(entries.into_boxed_slice()); |
61 | 61 | ||
62 | for (child_idx, subtree) in children { | 62 | for (child_idx, (subtree, tt)) in children { |
63 | let idx = TokenBuffer::new_inner( | 63 | let idx = TokenBuffer::new_inner( |
64 | &subtree.token_trees, | 64 | &subtree.token_trees, |
65 | buffers, | 65 | buffers, |
66 | Some(EntryPtr(EntryId(res), child_idx + 1)), | 66 | Some(EntryPtr(EntryId(res), child_idx + 1)), |
67 | ); | 67 | ); |
68 | buffers[res].as_mut()[child_idx] = Entry::Subtree(subtree, EntryId(idx)); | 68 | buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, EntryId(idx)); |
69 | } | 69 | } |
70 | 70 | ||
71 | res | 71 | res |
@@ -86,7 +86,7 @@ impl TokenBuffer { | |||
86 | /// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125 | 86 | /// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125 |
87 | #[derive(Copy, Clone, Debug)] | 87 | #[derive(Copy, Clone, Debug)] |
88 | pub struct Cursor<'a> { | 88 | pub struct Cursor<'a> { |
89 | buffer: &'a TokenBuffer, | 89 | buffer: &'a TokenBuffer<'a>, |
90 | ptr: EntryPtr, | 90 | ptr: EntryPtr, |
91 | } | 91 | } |
92 | 92 | ||
@@ -113,7 +113,7 @@ impl<'a> Cursor<'a> { | |||
113 | match self.entry() { | 113 | match self.entry() { |
114 | Some(Entry::End(Some(ptr))) => { | 114 | Some(Entry::End(Some(ptr))) => { |
115 | let idx = ptr.1; | 115 | let idx = ptr.1; |
116 | if let Some(Entry::Subtree(subtree, _)) = | 116 | if let Some(Entry::Subtree(TokenTree::Subtree(subtree), _)) = |
117 | self.buffer.entry(&EntryPtr(ptr.0, idx - 1)) | 117 | self.buffer.entry(&EntryPtr(ptr.0, idx - 1)) |
118 | { | 118 | { |
119 | return Some(subtree); | 119 | return Some(subtree); |
@@ -125,7 +125,7 @@ impl<'a> Cursor<'a> { | |||
125 | } | 125 | } |
126 | } | 126 | } |
127 | 127 | ||
128 | fn entry(self) -> Option<(&'a Entry)> { | 128 | fn entry(self) -> Option<(&'a Entry<'a>)> { |
129 | self.buffer.entry(&self.ptr) | 129 | self.buffer.entry(&self.ptr) |
130 | } | 130 | } |
131 | 131 | ||
@@ -141,10 +141,10 @@ impl<'a> Cursor<'a> { | |||
141 | } | 141 | } |
142 | 142 | ||
143 | /// If the cursor is pointing at a `TokenTree`, returns it | 143 | /// If the cursor is pointing at a `TokenTree`, returns it |
144 | pub fn token_tree(self) -> Option<(TokenTree)> { | 144 | pub fn token_tree(self) -> Option<&'a TokenTree> { |
145 | match self.entry() { | 145 | match self.entry() { |
146 | Some(Entry::Leaf(leaf)) => Some(leaf.clone().into()), | 146 | Some(Entry::Leaf(tt)) => Some(tt), |
147 | Some(Entry::Subtree(subtree, _)) => Some(subtree.clone().into()), | 147 | Some(Entry::Subtree(tt, _)) => Some(tt), |
148 | Some(Entry::End(_)) => None, | 148 | Some(Entry::End(_)) => None, |
149 | None => None, | 149 | None => None, |
150 | } | 150 | } |