aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crates/mbe/src/subtree_source.rs148
1 files changed, 60 insertions, 88 deletions
diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs
index 36d6f1038..d7433bd35 100644
--- a/crates/mbe/src/subtree_source.rs
+++ b/crates/mbe/src/subtree_source.rs
@@ -1,131 +1,104 @@
1//! FIXME: write short doc here 1//! FIXME: write short doc here
2 2
3use parser::{Token, TokenSource}; 3use parser::{Token, TokenSource};
4use std::cell::{Cell, Ref, RefCell};
5use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; 4use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
6use tt::buffer::{Cursor, TokenBuffer}; 5use tt::buffer::TokenBuffer;
7 6
8#[derive(Debug, Clone, Eq, PartialEq)] 7#[derive(Debug, Clone, Eq, PartialEq)]
9struct TtToken { 8struct TtToken {
10 kind: SyntaxKind, 9 tt: Token,
11 is_joint_to_next: bool,
12 text: SmolStr, 10 text: SmolStr,
13} 11}
14 12
15pub(crate) struct SubtreeTokenSource<'a> { 13pub(crate) struct SubtreeTokenSource {
16 cached_cursor: Cell<Cursor<'a>>, 14 cached: Vec<TtToken>,
17 cached: RefCell<Vec<Option<TtToken>>>,
18 curr: (Token, usize), 15 curr: (Token, usize),
19} 16}
20 17
21impl<'a> SubtreeTokenSource<'a> { 18impl<'a> SubtreeTokenSource {
22 // Helper function used in test 19 // Helper function used in test
23 #[cfg(test)] 20 #[cfg(test)]
24 pub(crate) fn text(&self) -> SmolStr { 21 pub(crate) fn text(&self) -> SmolStr {
25 match *self.get(self.curr.1) { 22 match self.cached.get(self.curr.1) {
26 Some(ref tt) => tt.text.clone(), 23 Some(ref tt) => tt.text.clone(),
27 _ => SmolStr::new(""), 24 _ => SmolStr::new(""),
28 } 25 }
29 } 26 }
30} 27}
31 28
32impl<'a> SubtreeTokenSource<'a> { 29impl<'a> SubtreeTokenSource {
33 pub(crate) fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> { 30 pub(crate) fn new(buffer: &TokenBuffer) -> SubtreeTokenSource {
34 let cursor = buffer.begin(); 31 let mut current = buffer.begin();
32 let mut cached = Vec::with_capacity(100);
35 33
36 let mut res = SubtreeTokenSource { 34 while !current.eof() {
37 curr: (Token { kind: EOF, is_jointed_to_next: false }, 0), 35 let cursor = current;
38 cached_cursor: Cell::new(cursor), 36 let tt = cursor.token_tree();
39 cached: RefCell::new(Vec::with_capacity(10)),
40 };
41 res.curr = (res.mk_token(0), 0);
42 res
43 }
44 37
45 fn mk_token(&self, pos: usize) -> Token { 38 // Check if it is lifetime
46 match *self.get(pos) { 39 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
47 Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
48 None => Token { kind: EOF, is_jointed_to_next: false },
49 }
50 }
51
52 fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
53 fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
54 let tkn = c.token_tree();
55
56 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tkn {
57 if punct.char == '\'' { 40 if punct.char == '\'' {
58 let next = c.bump(); 41 let next = cursor.bump();
59 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) = 42 if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(ident), _)) =
60 next.token_tree() 43 next.token_tree()
61 { 44 {
62 let res_cursor = next.bump(); 45 let text = SmolStr::new("'".to_string() + &ident.text);
63 let text = SmolStr::new("'".to_string() + &ident.to_string()); 46 cached.push(TtToken {
64 47 tt: Token { kind: LIFETIME_IDENT, is_jointed_to_next: false },
65 return Some((res_cursor, text)); 48 text,
49 });
50 current = next.bump();
51 continue;
66 } else { 52 } else {
67 panic!("Next token must be ident : {:#?}", next.token_tree()); 53 panic!("Next token must be ident : {:#?}", next.token_tree());
68 } 54 }
69 } 55 }
70 } 56 }
71 57
72 None 58 current = match tt {
73 } 59 Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
74 60 cached.push(convert_leaf(&leaf));
75 if pos < self.cached.borrow().len() { 61 cursor.bump()
76 return Ref::map(self.cached.borrow(), |c| &c[pos]);
77 }
78
79 {
80 let mut cached = self.cached.borrow_mut();
81 while pos >= cached.len() {
82 let cursor = self.cached_cursor.get();
83 if cursor.eof() {
84 cached.push(None);
85 continue;
86 } 62 }
87 63 Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
88 if let Some((curr, text)) = is_lifetime(cursor) { 64 cached.push(convert_delim(subtree.delimiter_kind(), false));
89 cached.push(Some(TtToken { 65 cursor.subtree().unwrap()
90 kind: LIFETIME_IDENT,
91 is_joint_to_next: false,
92 text,
93 }));
94 self.cached_cursor.set(curr);
95 continue;
96 } 66 }
97 67 None => {
98 match cursor.token_tree() { 68 if let Some(subtree) = cursor.end() {
99 Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { 69 cached.push(convert_delim(subtree.delimiter_kind(), true));
100 cached.push(Some(convert_leaf(&leaf))); 70 cursor.bump()
101 self.cached_cursor.set(cursor.bump()); 71 } else {
102 } 72 continue;
103 Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
104 self.cached_cursor.set(cursor.subtree().unwrap());
105 cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
106 }
107 None => {
108 if let Some(subtree) = cursor.end() {
109 cached.push(Some(convert_delim(subtree.delimiter_kind(), true)));
110 self.cached_cursor.set(cursor.bump());
111 }
112 } 73 }
113 } 74 }
114 } 75 };
115 } 76 }
116 77
117 Ref::map(self.cached.borrow(), |c| &c[pos]) 78 let mut res = SubtreeTokenSource {
79 curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
80 cached,
81 };
82 res.curr = (res.token(0), 0);
83 res
84 }
85
86 fn token(&self, pos: usize) -> Token {
87 match self.cached.get(pos) {
88 Some(it) => it.tt,
89 None => Token { kind: EOF, is_jointed_to_next: false },
90 }
118 } 91 }
119} 92}
120 93
121impl<'a> TokenSource for SubtreeTokenSource<'a> { 94impl<'a> TokenSource for SubtreeTokenSource {
122 fn current(&self) -> Token { 95 fn current(&self) -> Token {
123 self.curr.0 96 self.curr.0
124 } 97 }
125 98
126 /// Lookahead n token 99 /// Lookahead n token
127 fn lookahead_nth(&self, n: usize) -> Token { 100 fn lookahead_nth(&self, n: usize) -> Token {
128 self.mk_token(self.curr.1 + n) 101 self.token(self.curr.1 + n)
129 } 102 }
130 103
131 /// bump cursor to next token 104 /// bump cursor to next token
@@ -133,13 +106,12 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
133 if self.current().kind == EOF { 106 if self.current().kind == EOF {
134 return; 107 return;
135 } 108 }
136 109 self.curr = (self.token(self.curr.1 + 1), self.curr.1 + 1);
137 self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1);
138 } 110 }
139 111
140 /// Is the current token a specified keyword? 112 /// Is the current token a specified keyword?
141 fn is_keyword(&self, kw: &str) -> bool { 113 fn is_keyword(&self, kw: &str) -> bool {
142 match *self.get(self.curr.1) { 114 match self.cached.get(self.curr.1) {
143 Some(ref t) => t.text == *kw, 115 Some(ref t) => t.text == *kw,
144 _ => false, 116 _ => false,
145 } 117 }
@@ -157,7 +129,7 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
157 let idx = closing as usize; 129 let idx = closing as usize;
158 let kind = kinds[idx]; 130 let kind = kinds[idx];
159 let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" }; 131 let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" };
160 TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) } 132 TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) }
161} 133}
162 134
163fn convert_literal(l: &tt::Literal) -> TtToken { 135fn convert_literal(l: &tt::Literal) -> TtToken {
@@ -171,7 +143,7 @@ fn convert_literal(l: &tt::Literal) -> TtToken {
171 }) 143 })
172 .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); 144 .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
173 145
174 TtToken { kind, is_joint_to_next: false, text: l.text.clone() } 146 TtToken { tt: Token { kind, is_jointed_to_next: false }, text: l.text.clone() }
175} 147}
176 148
177fn convert_ident(ident: &tt::Ident) -> TtToken { 149fn convert_ident(ident: &tt::Ident) -> TtToken {
@@ -182,7 +154,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
182 _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT), 154 _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
183 }; 155 };
184 156
185 TtToken { kind, is_joint_to_next: false, text: ident.text.clone() } 157 TtToken { tt: Token { kind, is_jointed_to_next: false }, text: ident.text.clone() }
186} 158}
187 159
188fn convert_punct(p: tt::Punct) -> TtToken { 160fn convert_punct(p: tt::Punct) -> TtToken {
@@ -196,7 +168,7 @@ fn convert_punct(p: tt::Punct) -> TtToken {
196 let s: &str = p.char.encode_utf8(&mut buf); 168 let s: &str = p.char.encode_utf8(&mut buf);
197 SmolStr::new(s) 169 SmolStr::new(s)
198 }; 170 };
199 TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text } 171 TtToken { tt: Token { kind, is_jointed_to_next: p.spacing == tt::Spacing::Joint }, text }
200} 172}
201 173
202fn convert_leaf(leaf: &tt::Leaf) -> TtToken { 174fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
@@ -210,6 +182,7 @@ fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
210#[cfg(test)] 182#[cfg(test)]
211mod tests { 183mod tests {
212 use super::{convert_literal, TtToken}; 184 use super::{convert_literal, TtToken};
185 use parser::Token;
213 use syntax::{SmolStr, SyntaxKind}; 186 use syntax::{SmolStr, SyntaxKind};
214 187
215 #[test] 188 #[test]
@@ -220,8 +193,7 @@ mod tests {
220 text: SmolStr::new("-42.0") 193 text: SmolStr::new("-42.0")
221 }), 194 }),
222 TtToken { 195 TtToken {
223 kind: SyntaxKind::FLOAT_NUMBER, 196 tt: Token { kind: SyntaxKind::FLOAT_NUMBER, is_jointed_to_next: false },
224 is_joint_to_next: false,
225 text: SmolStr::new("-42.0") 197 text: SmolStr::new("-42.0")
226 } 198 }
227 ); 199 );