aboutsummaryrefslogtreecommitdiff
path: root/crates
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2021-01-04 16:22:42 +0000
committerEdwin Cheng <[email protected]>2021-01-04 18:00:46 +0000
commitaf3d75ad2e760dc885f54e6179543718ef8f141f (patch)
tree821093073f61fbfb2d3d31304dfe0eacbef0bcf0 /crates
parentf1ffd14922d2dc885ba6441ca8380f7d4cb75269 (diff)
Refactor TokenBuffer for reduc cloning
Diffstat (limited to 'crates')
-rw-r--r--crates/hir_expand/src/proc_macro.rs10
-rw-r--r--crates/mbe/src/mbe_expander/matcher.rs6
-rw-r--r--crates/mbe/src/subtree_source.rs10
-rw-r--r--crates/mbe/src/syntax_bridge.rs21
-rw-r--r--crates/tt/src/buffer.rs105
5 files changed, 99 insertions, 53 deletions
diff --git a/crates/hir_expand/src/proc_macro.rs b/crates/hir_expand/src/proc_macro.rs
index 7c77f6ce0..1923daca5 100644
--- a/crates/hir_expand/src/proc_macro.rs
+++ b/crates/hir_expand/src/proc_macro.rs
@@ -58,7 +58,7 @@ impl ProcMacroExpander {
58} 58}
59 59
60fn eat_punct(cursor: &mut Cursor, c: char) -> bool { 60fn eat_punct(cursor: &mut Cursor, c: char) -> bool {
61 if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = cursor.token_tree() { 61 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = cursor.token_tree() {
62 if punct.char == c { 62 if punct.char == c {
63 *cursor = cursor.bump(); 63 *cursor = cursor.bump();
64 return true; 64 return true;
@@ -68,7 +68,7 @@ fn eat_punct(cursor: &mut Cursor, c: char) -> bool {
68} 68}
69 69
70fn eat_subtree(cursor: &mut Cursor, kind: tt::DelimiterKind) -> bool { 70fn eat_subtree(cursor: &mut Cursor, kind: tt::DelimiterKind) -> bool {
71 if let Some(tt::TokenTree::Subtree(subtree)) = cursor.token_tree() { 71 if let Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) = cursor.token_tree() {
72 if Some(kind) == subtree.delimiter_kind() { 72 if Some(kind) == subtree.delimiter_kind() {
73 *cursor = cursor.bump_subtree(); 73 *cursor = cursor.bump_subtree();
74 return true; 74 return true;
@@ -78,7 +78,7 @@ fn eat_subtree(cursor: &mut Cursor, kind: tt::DelimiterKind) -> bool {
78} 78}
79 79
80fn eat_ident(cursor: &mut Cursor, t: &str) -> bool { 80fn eat_ident(cursor: &mut Cursor, t: &str) -> bool {
81 if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = cursor.token_tree() { 81 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) = cursor.token_tree() {
82 if t == ident.text.as_str() { 82 if t == ident.text.as_str() {
83 *cursor = cursor.bump(); 83 *cursor = cursor.bump();
84 return true; 84 return true;
@@ -88,7 +88,7 @@ fn eat_ident(cursor: &mut Cursor, t: &str) -> bool {
88} 88}
89 89
90fn remove_derive_attrs(tt: &tt::Subtree) -> Option<tt::Subtree> { 90fn remove_derive_attrs(tt: &tt::Subtree) -> Option<tt::Subtree> {
91 let buffer = TokenBuffer::new(&tt.token_trees); 91 let buffer = TokenBuffer::from_tokens(&tt.token_trees);
92 let mut p = buffer.begin(); 92 let mut p = buffer.begin();
93 let mut result = tt::Subtree::default(); 93 let mut result = tt::Subtree::default();
94 94
@@ -106,7 +106,7 @@ fn remove_derive_attrs(tt: &tt::Subtree) -> Option<tt::Subtree> {
106 } 106 }
107 } 107 }
108 108
109 result.token_trees.push(curr.token_tree()?.clone()); 109 result.token_trees.push(curr.token_tree()?.cloned());
110 p = curr.bump(); 110 p = curr.bump();
111 } 111 }
112 112
diff --git a/crates/mbe/src/mbe_expander/matcher.rs b/crates/mbe/src/mbe_expander/matcher.rs
index ab5f87c48..fdc8844ce 100644
--- a/crates/mbe/src/mbe_expander/matcher.rs
+++ b/crates/mbe/src/mbe_expander/matcher.rs
@@ -309,7 +309,7 @@ impl<'a> TtIter<'a> {
309 } 309 }
310 } 310 }
311 311
312 let buffer = TokenBuffer::new(&self.inner.as_slice()); 312 let buffer = TokenBuffer::from_tokens(&self.inner.as_slice());
313 let mut src = SubtreeTokenSource::new(&buffer); 313 let mut src = SubtreeTokenSource::new(&buffer);
314 let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false }; 314 let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
315 315
@@ -336,11 +336,11 @@ impl<'a> TtIter<'a> {
336 err = Some(err!("no tokens consumed")); 336 err = Some(err!("no tokens consumed"));
337 } 337 }
338 let res = match res.len() { 338 let res = match res.len() {
339 1 => Some(res[0].clone()), 339 1 => Some(res[0].cloned()),
340 0 => None, 340 0 => None,
341 _ => Some(tt::TokenTree::Subtree(tt::Subtree { 341 _ => Some(tt::TokenTree::Subtree(tt::Subtree {
342 delimiter: None, 342 delimiter: None,
343 token_trees: res.into_iter().cloned().collect(), 343 token_trees: res.into_iter().map(|it| it.cloned()).collect(),
344 })), 344 })),
345 }; 345 };
346 ExpandResult { value: res, err } 346 ExpandResult { value: res, err }
diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs
index d10d4b70e..36d6f1038 100644
--- a/crates/mbe/src/subtree_source.rs
+++ b/crates/mbe/src/subtree_source.rs
@@ -53,10 +53,12 @@ impl<'a> SubtreeTokenSource<'a> {
53 fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> { 53 fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
54 let tkn = c.token_tree(); 54 let tkn = c.token_tree();
55 55
56 if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn { 56 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tkn {
57 if punct.char == '\'' { 57 if punct.char == '\'' {
58 let next = c.bump(); 58 let next = c.bump();
59 if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() { 59 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
60 next.token_tree()
61 {
60 let res_cursor = next.bump(); 62 let res_cursor = next.bump();
61 let text = SmolStr::new("'".to_string() + &ident.to_string()); 63 let text = SmolStr::new("'".to_string() + &ident.to_string());
62 64
@@ -94,11 +96,11 @@ impl<'a> SubtreeTokenSource<'a> {
94 } 96 }
95 97
96 match cursor.token_tree() { 98 match cursor.token_tree() {
97 Some(tt::TokenTree::Leaf(leaf)) => { 99 Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
98 cached.push(Some(convert_leaf(&leaf))); 100 cached.push(Some(convert_leaf(&leaf)));
99 self.cached_cursor.set(cursor.bump()); 101 self.cached_cursor.set(cursor.bump());
100 } 102 }
101 Some(tt::TokenTree::Subtree(subtree)) => { 103 Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
102 self.cached_cursor.set(cursor.subtree().unwrap()); 104 self.cached_cursor.set(cursor.subtree().unwrap());
103 cached.push(Some(convert_delim(subtree.delimiter_kind(), false))); 105 cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
104 } 106 }
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
index 423a4934e..671036e1c 100644
--- a/crates/mbe/src/syntax_bridge.rs
+++ b/crates/mbe/src/syntax_bridge.rs
@@ -70,15 +70,12 @@ pub fn token_tree_to_syntax_node(
70 tt: &tt::Subtree, 70 tt: &tt::Subtree,
71 fragment_kind: FragmentKind, 71 fragment_kind: FragmentKind,
72) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> { 72) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> {
73 let tmp; 73 let buffer = match tt {
74 let tokens = match tt { 74 tt::Subtree { delimiter: None, token_trees } => {
75 tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(), 75 TokenBuffer::from_tokens(token_trees.as_slice())
76 _ => {
77 tmp = [tt.clone().into()];
78 &tmp[..]
79 } 76 }
77 _ => TokenBuffer::from_subtree(tt),
80 }; 78 };
81 let buffer = TokenBuffer::new(&tokens);
82 let mut token_source = SubtreeTokenSource::new(&buffer); 79 let mut token_source = SubtreeTokenSource::new(&buffer);
83 let mut tree_sink = TtTreeSink::new(buffer.begin()); 80 let mut tree_sink = TtTreeSink::new(buffer.begin());
84 parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind); 81 parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind);
@@ -631,7 +628,7 @@ impl<'a> TreeSink for TtTreeSink<'a> {
631 } 628 }
632 last = self.cursor; 629 last = self.cursor;
633 let text: &str = match self.cursor.token_tree() { 630 let text: &str = match self.cursor.token_tree() {
634 Some(tt::TokenTree::Leaf(leaf)) => { 631 Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
635 // Mark the range if needed 632 // Mark the range if needed
636 let (text, id) = match leaf { 633 let (text, id) = match leaf {
637 tt::Leaf::Ident(ident) => (&ident.text, ident.id), 634 tt::Leaf::Ident(ident) => (&ident.text, ident.id),
@@ -650,7 +647,7 @@ impl<'a> TreeSink for TtTreeSink<'a> {
650 self.cursor = self.cursor.bump(); 647 self.cursor = self.cursor.bump();
651 text 648 text
652 } 649 }
653 Some(tt::TokenTree::Subtree(subtree)) => { 650 Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
654 self.cursor = self.cursor.subtree().unwrap(); 651 self.cursor = self.cursor.subtree().unwrap();
655 if let Some(id) = subtree.delimiter.map(|it| it.id) { 652 if let Some(id) = subtree.delimiter.map(|it| it.id) {
656 self.open_delims.insert(id, self.text_pos); 653 self.open_delims.insert(id, self.text_pos);
@@ -684,8 +681,8 @@ impl<'a> TreeSink for TtTreeSink<'a> {
684 // Add whitespace between adjoint puncts 681 // Add whitespace between adjoint puncts
685 let next = last.bump(); 682 let next = last.bump();
686 if let ( 683 if let (
687 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))), 684 Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(curr), _)),
688 Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))), 685 Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(_), _)),
689 ) = (last.token_tree(), next.token_tree()) 686 ) = (last.token_tree(), next.token_tree())
690 { 687 {
691 // Note: We always assume the semi-colon would be the last token in 688 // Note: We always assume the semi-colon would be the last token in
@@ -744,7 +741,7 @@ mod tests {
744 ) 741 )
745 .expand_tt("literals!(foo);"); 742 .expand_tt("literals!(foo);");
746 let tts = &[expansion.into()]; 743 let tts = &[expansion.into()];
747 let buffer = tt::buffer::TokenBuffer::new(tts); 744 let buffer = tt::buffer::TokenBuffer::from_tokens(tts);
748 let mut tt_src = SubtreeTokenSource::new(&buffer); 745 let mut tt_src = SubtreeTokenSource::new(&buffer);
749 let mut tokens = vec![]; 746 let mut tokens = vec![];
750 while tt_src.current().kind != EOF { 747 while tt_src.current().kind != EOF {
diff --git a/crates/tt/src/buffer.rs b/crates/tt/src/buffer.rs
index 02c771f70..3606c887d 100644
--- a/crates/tt/src/buffer.rs
+++ b/crates/tt/src/buffer.rs
@@ -1,6 +1,6 @@
1//! FIXME: write short doc here 1//! FIXME: write short doc here
2 2
3use crate::{Subtree, TokenTree}; 3use crate::{Leaf, Subtree, TokenTree};
4 4
5#[derive(Copy, Clone, Debug, Eq, PartialEq)] 5#[derive(Copy, Clone, Debug, Eq, PartialEq)]
6struct EntryId(usize); 6struct EntryId(usize);
@@ -13,7 +13,7 @@ struct EntryPtr(EntryId, usize);
13#[derive(Debug)] 13#[derive(Debug)]
14enum Entry<'t> { 14enum Entry<'t> {
15 // Mimicking types from proc-macro. 15 // Mimicking types from proc-macro.
16 Subtree(&'t TokenTree, EntryId), 16 Subtree(Option<&'t TokenTree>, &'t Subtree, EntryId),
17 Leaf(&'t TokenTree), 17 Leaf(&'t TokenTree),
18 // End entries contain a pointer to the entry from the containing 18 // End entries contain a pointer to the entry from the containing
19 // token tree, or None if this is the outermost level. 19 // token tree, or None if this is the outermost level.
@@ -27,37 +27,64 @@ pub struct TokenBuffer<'t> {
27 buffers: Vec<Box<[Entry<'t>]>>, 27 buffers: Vec<Box<[Entry<'t>]>>,
28} 28}
29 29
30impl<'t> TokenBuffer<'t> { 30trait TokenList<'a> {
31 pub fn new(tokens: &'t [TokenTree]) -> TokenBuffer<'t> { 31 fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>);
32 let mut buffers = vec![]; 32}
33
34 let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
35 assert_eq!(idx, 0);
36
37 TokenBuffer { buffers }
38 }
39 33
40 fn new_inner( 34impl<'a> TokenList<'a> for &'a [TokenTree] {
41 tokens: &'t [TokenTree], 35 fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
42 buffers: &mut Vec<Box<[Entry<'t>]>>,
43 next: Option<EntryPtr>,
44 ) -> usize {
45 // Must contain everything in tokens and then the Entry::End 36 // Must contain everything in tokens and then the Entry::End
46 let start_capacity = tokens.len() + 1; 37 let start_capacity = self.len() + 1;
47 let mut entries = Vec::with_capacity(start_capacity); 38 let mut entries = Vec::with_capacity(start_capacity);
48 let mut children = vec![]; 39 let mut children = vec![];
49 40 for (idx, tt) in self.iter().enumerate() {
50 for (idx, tt) in tokens.iter().enumerate() {
51 match tt { 41 match tt {
52 TokenTree::Leaf(_) => { 42 TokenTree::Leaf(_) => {
53 entries.push(Entry::Leaf(tt)); 43 entries.push(Entry::Leaf(tt));
54 } 44 }
55 TokenTree::Subtree(subtree) => { 45 TokenTree::Subtree(subtree) => {
56 entries.push(Entry::End(None)); 46 entries.push(Entry::End(None));
57 children.push((idx, (subtree, tt))); 47 children.push((idx, (subtree, Some(tt))));
58 } 48 }
59 } 49 }
60 } 50 }
51 (children, entries)
52 }
53}
54
55impl<'a> TokenList<'a> for &'a Subtree {
56 fn entries(&self) -> (Vec<(usize, (&'a Subtree, Option<&'a TokenTree>))>, Vec<Entry<'a>>) {
57 // Must contain everything in tokens and then the Entry::End
58 let mut entries = vec![];
59 let mut children = vec![];
60 entries.push(Entry::End(None));
61 children.push((0usize, (*self, None)));
62 (children, entries)
63 }
64}
65
66impl<'t> TokenBuffer<'t> {
67 pub fn from_tokens(tokens: &'t [TokenTree]) -> TokenBuffer<'t> {
68 Self::new(tokens)
69 }
70
71 pub fn from_subtree(subtree: &'t Subtree) -> TokenBuffer<'t> {
72 Self::new(subtree)
73 }
74
75 fn new<T: TokenList<'t>>(tokens: T) -> TokenBuffer<'t> {
76 let mut buffers = vec![];
77 let idx = TokenBuffer::new_inner(tokens, &mut buffers, None);
78 assert_eq!(idx, 0);
79 TokenBuffer { buffers }
80 }
81
82 fn new_inner<T: TokenList<'t>>(
83 tokens: T,
84 buffers: &mut Vec<Box<[Entry<'t>]>>,
85 next: Option<EntryPtr>,
86 ) -> usize {
87 let (children, mut entries) = tokens.entries();
61 88
62 entries.push(Entry::End(next)); 89 entries.push(Entry::End(next));
63 let res = buffers.len(); 90 let res = buffers.len();
@@ -65,11 +92,11 @@ impl<'t> TokenBuffer<'t> {
65 92
66 for (child_idx, (subtree, tt)) in children { 93 for (child_idx, (subtree, tt)) in children {
67 let idx = TokenBuffer::new_inner( 94 let idx = TokenBuffer::new_inner(
68 &subtree.token_trees, 95 subtree.token_trees.as_slice(),
69 buffers, 96 buffers,
70 Some(EntryPtr(EntryId(res), child_idx + 1)), 97 Some(EntryPtr(EntryId(res), child_idx + 1)),
71 ); 98 );
72 buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, EntryId(idx)); 99 buffers[res].as_mut()[child_idx] = Entry::Subtree(tt, subtree, EntryId(idx));
73 } 100 }
74 101
75 res 102 res
@@ -87,6 +114,24 @@ impl<'t> TokenBuffer<'t> {
87 } 114 }
88} 115}
89 116
117#[derive(Debug)]
118pub enum TokenTreeRef<'a> {
119 Subtree(&'a Subtree, Option<&'a TokenTree>),
120 Leaf(&'a Leaf, &'a TokenTree),
121}
122
123impl<'a> TokenTreeRef<'a> {
124 pub fn cloned(&self) -> TokenTree {
125 match &self {
126 TokenTreeRef::Subtree(subtree, tt) => match tt {
127 Some(it) => (*it).clone(),
128 None => (*subtree).clone().into(),
129 },
130 TokenTreeRef::Leaf(_, tt) => (*tt).clone(),
131 }
132 }
133}
134
90/// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125 135/// A safe version of `Cursor` from `syn` crate https://github.com/dtolnay/syn/blob/6533607f91686545cb034d2838beea338d9d0742/src/buffer.rs#L125
91#[derive(Copy, Clone, Debug)] 136#[derive(Copy, Clone, Debug)]
92pub struct Cursor<'a> { 137pub struct Cursor<'a> {
@@ -114,12 +159,11 @@ impl<'a> Cursor<'a> {
114 match self.entry() { 159 match self.entry() {
115 Some(Entry::End(Some(ptr))) => { 160 Some(Entry::End(Some(ptr))) => {
116 let idx = ptr.1; 161 let idx = ptr.1;
117 if let Some(Entry::Subtree(TokenTree::Subtree(subtree), _)) = 162 if let Some(Entry::Subtree(_, subtree, _)) =
118 self.buffer.entry(&EntryPtr(ptr.0, idx - 1)) 163 self.buffer.entry(&EntryPtr(ptr.0, idx - 1))
119 { 164 {
120 return Some(subtree); 165 return Some(subtree);
121 } 166 }
122
123 None 167 None
124 } 168 }
125 _ => None, 169 _ => None,
@@ -134,7 +178,7 @@ impl<'a> Cursor<'a> {
134 /// a cursor into that subtree 178 /// a cursor into that subtree
135 pub fn subtree(self) -> Option<Cursor<'a>> { 179 pub fn subtree(self) -> Option<Cursor<'a>> {
136 match self.entry() { 180 match self.entry() {
137 Some(Entry::Subtree(_, entry_id)) => { 181 Some(Entry::Subtree(_, _, entry_id)) => {
138 Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0))) 182 Some(Cursor::create(self.buffer, EntryPtr(*entry_id, 0)))
139 } 183 }
140 _ => None, 184 _ => None,
@@ -142,10 +186,13 @@ impl<'a> Cursor<'a> {
142 } 186 }
143 187
144 /// If the cursor is pointing at a `TokenTree`, returns it 188 /// If the cursor is pointing at a `TokenTree`, returns it
145 pub fn token_tree(self) -> Option<&'a TokenTree> { 189 pub fn token_tree(self) -> Option<TokenTreeRef<'a>> {
146 match self.entry() { 190 match self.entry() {
147 Some(Entry::Leaf(tt)) => Some(tt), 191 Some(Entry::Leaf(tt)) => match tt {
148 Some(Entry::Subtree(tt, _)) => Some(tt), 192 TokenTree::Leaf(leaf) => Some(TokenTreeRef::Leaf(leaf, *tt)),
193 TokenTree::Subtree(subtree) => Some(TokenTreeRef::Subtree(subtree, Some(tt))),
194 },
195 Some(Entry::Subtree(tt, subtree, _)) => Some(TokenTreeRef::Subtree(subtree, *tt)),
149 Some(Entry::End(_)) => None, 196 Some(Entry::End(_)) => None,
150 None => None, 197 None => None,
151 } 198 }
@@ -172,7 +219,7 @@ impl<'a> Cursor<'a> {
172 /// a cursor into that subtree 219 /// a cursor into that subtree
173 pub fn bump_subtree(self) -> Cursor<'a> { 220 pub fn bump_subtree(self) -> Cursor<'a> {
174 match self.entry() { 221 match self.entry() {
175 Some(Entry::Subtree(_, _)) => self.subtree().unwrap(), 222 Some(Entry::Subtree(_, _, _)) => self.subtree().unwrap(),
176 _ => self.bump(), 223 _ => self.bump(),
177 } 224 }
178 } 225 }