diff options
Diffstat (limited to 'crates/ra_mbe/src/subtree_source.rs')
-rw-r--r-- | crates/ra_mbe/src/subtree_source.rs | 623 |
1 files changed, 379 insertions, 244 deletions
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs index d9ba5d3d0..9dd475f2c 100644 --- a/crates/ra_mbe/src/subtree_source.rs +++ b/crates/ra_mbe/src/subtree_source.rs | |||
@@ -1,7 +1,8 @@ | |||
1 | use ra_parser::{TokenSource}; | 1 | use ra_parser::{TokenSource}; |
2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; | 2 | use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*}; |
3 | use std::cell::{RefCell}; | ||
3 | 4 | ||
4 | #[derive(Debug)] | 5 | #[derive(Debug, Clone, Eq, PartialEq)] |
5 | struct TtToken { | 6 | struct TtToken { |
6 | pub kind: SyntaxKind, | 7 | pub kind: SyntaxKind, |
7 | pub is_joint_to_next: bool, | 8 | pub is_joint_to_next: bool, |
@@ -9,107 +10,319 @@ struct TtToken { | |||
9 | pub n_tokens: usize, | 10 | pub n_tokens: usize, |
10 | } | 11 | } |
11 | 12 | ||
12 | /// Querier let outside to query internal tokens as string | 13 | #[derive(Debug, Clone, Eq, PartialEq)] |
13 | pub(crate) struct Querier<'a> { | 14 | enum WalkIndex { |
14 | src: &'a SubtreeTokenSource<'a>, | 15 | DelimiterBegin(Option<TtToken>), |
16 | Token(usize, Option<TtToken>), | ||
17 | DelimiterEnd(Option<TtToken>), | ||
18 | Eof, | ||
15 | } | 19 | } |
16 | 20 | ||
17 | impl<'a> Querier<'a> { | 21 | impl<'a> SubTreeWalker<'a> { |
18 | pub(crate) fn token(&self, uidx: usize) -> (SyntaxKind, &SmolStr) { | 22 | fn new(subtree: &tt::Subtree) -> SubTreeWalker { |
19 | let tkn = &self.src.tokens[uidx]; | 23 | let mut res = SubTreeWalker { |
20 | (tkn.kind, &tkn.text) | 24 | pos: 0, |
25 | stack: vec![], | ||
26 | idx: WalkIndex::Eof, | ||
27 | last_steps: vec![], | ||
28 | subtree, | ||
29 | }; | ||
30 | |||
31 | res.reset(); | ||
32 | res | ||
33 | } | ||
34 | |||
35 | fn reset(&mut self) { | ||
36 | self.pos = 0; | ||
37 | self.stack = vec![(self.subtree, None)]; | ||
38 | self.idx = WalkIndex::DelimiterBegin(convert_delim(self.subtree.delimiter, false)); | ||
39 | self.last_steps = vec![]; | ||
40 | |||
41 | while self.is_empty_delimiter() { | ||
42 | self.forward_unchecked(); | ||
43 | } | ||
44 | } | ||
45 | |||
46 | // This funciton will fast forward the pos cursor, | ||
47 | // Such that backward will stop at `start_pos` point | ||
48 | fn start_from_nth(&mut self, start_pos: usize) { | ||
49 | self.reset(); | ||
50 | self.pos = start_pos; | ||
51 | self.idx = self.walk_token(start_pos, false); | ||
52 | |||
53 | while self.is_empty_delimiter() { | ||
54 | self.forward_unchecked(); | ||
55 | } | ||
56 | } | ||
57 | |||
58 | fn current(&self) -> Option<&TtToken> { | ||
59 | match &self.idx { | ||
60 | WalkIndex::DelimiterBegin(t) => t.as_ref(), | ||
61 | WalkIndex::Token(_, t) => t.as_ref(), | ||
62 | WalkIndex::DelimiterEnd(t) => t.as_ref(), | ||
63 | WalkIndex::Eof => None, | ||
64 | } | ||
65 | } | ||
66 | |||
67 | fn is_empty_delimiter(&self) -> bool { | ||
68 | match &self.idx { | ||
69 | WalkIndex::DelimiterBegin(None) => true, | ||
70 | WalkIndex::DelimiterEnd(None) => true, | ||
71 | _ => false, | ||
72 | } | ||
73 | } | ||
74 | |||
75 | fn backward(&mut self) { | ||
76 | if self.last_steps.is_empty() { | ||
77 | return; | ||
78 | } | ||
79 | self.pos -= 1; | ||
80 | loop { | ||
81 | self.backward_unchecked(); | ||
82 | // Skip Empty delimiter | ||
83 | if self.last_steps.is_empty() || !self.is_empty_delimiter() { | ||
84 | break; | ||
85 | } | ||
86 | } | ||
87 | } | ||
88 | |||
89 | fn backward_unchecked(&mut self) { | ||
90 | if self.last_steps.is_empty() { | ||
91 | return; | ||
92 | } | ||
93 | |||
94 | let last_step = self.last_steps.pop().unwrap(); | ||
95 | let do_walk_token = match self.idx { | ||
96 | WalkIndex::DelimiterBegin(_) => None, | ||
97 | WalkIndex::Token(u, _) => Some(u), | ||
98 | WalkIndex::DelimiterEnd(_) => { | ||
99 | let (top, _) = self.stack.last().unwrap(); | ||
100 | Some(top.token_trees.len()) | ||
101 | } | ||
102 | WalkIndex::Eof => None, | ||
103 | }; | ||
104 | |||
105 | self.idx = match do_walk_token { | ||
106 | Some(u) if last_step > u => WalkIndex::DelimiterBegin(convert_delim( | ||
107 | self.stack.last().unwrap().0.delimiter, | ||
108 | false, | ||
109 | )), | ||
110 | Some(u) => self.walk_token(u - last_step, true), | ||
111 | None => match self.idx { | ||
112 | WalkIndex::Eof => { | ||
113 | self.stack.push((self.subtree, None)); | ||
114 | WalkIndex::DelimiterEnd(convert_delim( | ||
115 | self.stack.last().unwrap().0.delimiter, | ||
116 | true, | ||
117 | )) | ||
118 | } | ||
119 | _ => { | ||
120 | let (_, last_top_idx) = self.stack.pop().unwrap(); | ||
121 | assert!(!self.stack.is_empty()); | ||
122 | |||
123 | match last_top_idx.unwrap() { | ||
124 | 0 => WalkIndex::DelimiterBegin(convert_delim( | ||
125 | self.stack.last().unwrap().0.delimiter, | ||
126 | false, | ||
127 | )), | ||
128 | c => self.walk_token(c - 1, true), | ||
129 | } | ||
130 | } | ||
131 | }, | ||
132 | }; | ||
133 | } | ||
134 | |||
135 | fn forward(&mut self) { | ||
136 | self.pos += 1; | ||
137 | loop { | ||
138 | self.forward_unchecked(); | ||
139 | if !self.is_empty_delimiter() { | ||
140 | break; | ||
141 | } | ||
142 | } | ||
143 | } | ||
144 | |||
145 | fn forward_unchecked(&mut self) { | ||
146 | if self.idx == WalkIndex::Eof { | ||
147 | return; | ||
148 | } | ||
149 | |||
150 | let step = self.current().map(|x| x.n_tokens).unwrap_or(1); | ||
151 | self.last_steps.push(step); | ||
152 | |||
153 | let do_walk_token = match self.idx { | ||
154 | WalkIndex::DelimiterBegin(_) => Some(0), | ||
155 | WalkIndex::Token(u, _) => Some(u + step), | ||
156 | WalkIndex::DelimiterEnd(_) => None, | ||
157 | _ => unreachable!(), | ||
158 | }; | ||
159 | |||
160 | let (top, _) = self.stack.last().unwrap(); | ||
161 | |||
162 | self.idx = match do_walk_token { | ||
163 | Some(u) if u >= top.token_trees.len() => { | ||
164 | WalkIndex::DelimiterEnd(convert_delim(self.stack.last().unwrap().0.delimiter, true)) | ||
165 | } | ||
166 | Some(u) => self.walk_token(u, false), | ||
167 | None => { | ||
168 | let (_, last_top_idx) = self.stack.pop().unwrap(); | ||
169 | match self.stack.last() { | ||
170 | Some(top) => match last_top_idx.unwrap() { | ||
171 | idx if idx + 1 >= top.0.token_trees.len() => { | ||
172 | WalkIndex::DelimiterEnd(convert_delim(top.0.delimiter, true)) | ||
173 | } | ||
174 | idx => self.walk_token(idx + 1, false), | ||
175 | }, | ||
176 | |||
177 | None => WalkIndex::Eof, | ||
178 | } | ||
179 | } | ||
180 | }; | ||
181 | } | ||
182 | |||
183 | fn walk_token(&mut self, pos: usize, backward: bool) -> WalkIndex { | ||
184 | let (top, _) = self.stack.last().unwrap(); | ||
185 | match &top.token_trees[pos] { | ||
186 | tt::TokenTree::Subtree(subtree) => { | ||
187 | self.stack.push((subtree, Some(pos))); | ||
188 | let delim = convert_delim(self.stack.last().unwrap().0.delimiter, backward); | ||
189 | if backward { | ||
190 | WalkIndex::DelimiterEnd(delim) | ||
191 | } else { | ||
192 | WalkIndex::DelimiterBegin(delim) | ||
193 | } | ||
194 | } | ||
195 | tt::TokenTree::Leaf(leaf) => WalkIndex::Token(pos, Some(self.walk_leaf(leaf, pos))), | ||
196 | } | ||
197 | } | ||
198 | |||
199 | fn walk_leaf(&mut self, leaf: &tt::Leaf, pos: usize) -> TtToken { | ||
200 | match leaf { | ||
201 | tt::Leaf::Literal(l) => convert_literal(l), | ||
202 | tt::Leaf::Ident(ident) => convert_ident(ident), | ||
203 | tt::Leaf::Punct(punct) => { | ||
204 | let (top, _) = self.stack.last().unwrap(); | ||
205 | convert_punct(punct, top, pos) | ||
206 | } | ||
207 | } | ||
21 | } | 208 | } |
22 | } | 209 | } |
23 | 210 | ||
24 | pub(crate) struct SubtreeTokenSource<'a> { | 211 | pub(crate) trait Querier { |
25 | tt_pos: usize, | 212 | fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr); |
26 | tokens: Vec<TtToken>, | ||
27 | subtree: &'a tt::Subtree, | ||
28 | } | 213 | } |
29 | 214 | ||
30 | impl<'a> SubtreeTokenSource<'a> { | 215 | // A wrapper class for ref cell |
31 | pub fn new(subtree: &tt::Subtree) -> SubtreeTokenSource { | 216 | pub(crate) struct WalkerOwner<'a> { |
32 | SubtreeTokenSource { tokens: TtTokenBuilder::build(subtree), tt_pos: 0, subtree } | 217 | walker: RefCell<SubTreeWalker<'a>>, |
218 | offset: usize, | ||
219 | } | ||
220 | |||
221 | impl<'a> WalkerOwner<'a> { | ||
222 | fn token_idx<'b>(&self, pos: usize) -> Option<TtToken> { | ||
223 | self.set_walker_pos(pos); | ||
224 | self.walker.borrow().current().cloned() | ||
33 | } | 225 | } |
34 | 226 | ||
35 | // Advance token source and skip the first delimiter | 227 | fn start_from_nth(&mut self, pos: usize) { |
36 | pub fn advance(&mut self, n_token: usize, skip_first_delimiter: bool) { | 228 | self.offset = pos; |
37 | if skip_first_delimiter { | 229 | self.walker.borrow_mut().start_from_nth(pos); |
38 | self.tt_pos += 1; | 230 | } |
39 | } | ||
40 | 231 | ||
41 | // Matching `TtToken` cursor to `tt::TokenTree` cursor | 232 | fn set_walker_pos(&self, mut pos: usize) { |
42 | // It is because TtToken is not One to One mapping to tt::Token | 233 | pos += self.offset; |
43 | // There are 3 case (`TtToken` <=> `tt::TokenTree`) : | 234 | let mut walker = self.walker.borrow_mut(); |
44 | // * One to One => ident, single char punch | 235 | while pos > walker.pos { |
45 | // * Many to One => `tt::TokenTree::SubTree` | 236 | walker.forward(); |
46 | // * One to Many => multibyte punct | 237 | } |
47 | // | 238 | while pos < walker.pos { |
48 | // Such that we cannot simpliy advance the cursor | 239 | walker.backward(); |
49 | // We have to bump it one by one | ||
50 | let mut pos = 0; | ||
51 | while pos < n_token { | ||
52 | pos += self.bump(&self.subtree.token_trees[pos]); | ||
53 | } | 240 | } |
241 | assert!(pos == walker.pos); | ||
54 | } | 242 | } |
55 | 243 | ||
56 | pub fn querier(&self) -> Querier { | 244 | fn new(subtree: &'a tt::Subtree) -> Self { |
57 | Querier { src: self } | 245 | WalkerOwner { walker: RefCell::new(SubTreeWalker::new(subtree)), offset: 0 } |
58 | } | 246 | } |
59 | 247 | ||
60 | pub(crate) fn bump_n( | 248 | fn collect_token_tree(&mut self, n: usize) -> Vec<&tt::TokenTree> { |
61 | &mut self, | 249 | self.start_from_nth(self.offset); |
62 | n_tt_tokens: usize, | 250 | |
63 | token_pos: &mut usize, | ||
64 | ) -> Vec<&tt::TokenTree> { | ||
65 | let mut res = vec![]; | 251 | let mut res = vec![]; |
66 | // Matching `TtToken` cursor to `tt::TokenTree` cursor | 252 | let mut walker = self.walker.borrow_mut(); |
67 | // It is because TtToken is not One to One mapping to tt::Token | 253 | |
68 | // There are 3 case (`TtToken` <=> `tt::TokenTree`) : | 254 | while walker.pos - self.offset < n { |
69 | // * One to One => ident, single char punch | 255 | if let WalkIndex::Token(u, tt) = &walker.idx { |
70 | // * Many to One => `tt::TokenTree::SubTree` | 256 | if walker.stack.len() == 1 { |
71 | // * One to Many => multibyte punct | 257 | // We only collect the topmost child |
72 | // | 258 | res.push(&walker.stack[0].0.token_trees[*u]); |
73 | // Such that we cannot simpliy advance the cursor | 259 | if let Some(tt) = tt { |
74 | // We have to bump it one by one | 260 | for i in 0..tt.n_tokens - 1 { |
75 | let next_pos = self.tt_pos + n_tt_tokens; | 261 | res.push(&walker.stack[0].0.token_trees[u + i]); |
76 | 262 | } | |
77 | while self.tt_pos < next_pos { | 263 | } |
78 | let current = &self.subtree.token_trees[*token_pos]; | 264 | } |
79 | let n = self.bump(current); | 265 | } |
80 | res.extend((0..n).map(|i| &self.subtree.token_trees[*token_pos + i])); | 266 | |
81 | *token_pos += n; | 267 | walker.forward(); |
82 | } | 268 | } |
83 | 269 | ||
84 | res | 270 | res |
85 | } | 271 | } |
272 | } | ||
273 | |||
274 | impl<'a> Querier for WalkerOwner<'a> { | ||
275 | fn token(&self, uidx: usize) -> (SyntaxKind, SmolStr) { | ||
276 | let tkn = self.token_idx(uidx).unwrap(); | ||
277 | (tkn.kind, tkn.text) | ||
278 | } | ||
279 | } | ||
280 | |||
281 | pub(crate) struct SubtreeTokenSource<'a> { | ||
282 | walker: WalkerOwner<'a>, | ||
283 | } | ||
284 | |||
285 | impl<'a> SubtreeTokenSource<'a> { | ||
286 | pub fn new(subtree: &tt::Subtree) -> SubtreeTokenSource { | ||
287 | SubtreeTokenSource { walker: WalkerOwner::new(subtree) } | ||
288 | } | ||
289 | |||
290 | pub fn start_from_nth(&mut self, n: usize) { | ||
291 | self.walker.start_from_nth(n); | ||
292 | } | ||
86 | 293 | ||
87 | fn count(&self, tt: &tt::TokenTree) -> usize { | 294 | pub fn querier<'b>(&'a self) -> &'b WalkerOwner<'a> |
88 | assert!(!self.tokens.is_empty()); | 295 | where |
89 | TtTokenBuilder::count_tt_tokens(tt, None) | 296 | 'a: 'b, |
297 | { | ||
298 | &self.walker | ||
90 | } | 299 | } |
91 | 300 | ||
92 | fn bump(&mut self, tt: &tt::TokenTree) -> usize { | 301 | pub(crate) fn bump_n( |
93 | let cur = &self.tokens[self.tt_pos]; | 302 | &mut self, |
94 | let n_tokens = cur.n_tokens; | 303 | parsed_tokens: usize, |
95 | self.tt_pos += self.count(tt); | 304 | cursor_pos: &mut usize, |
96 | n_tokens | 305 | ) -> Vec<&tt::TokenTree> { |
306 | let res = self.walker.collect_token_tree(parsed_tokens); | ||
307 | *cursor_pos += res.len(); | ||
308 | |||
309 | res | ||
97 | } | 310 | } |
98 | } | 311 | } |
99 | 312 | ||
100 | impl<'a> TokenSource for SubtreeTokenSource<'a> { | 313 | impl<'a> TokenSource for SubtreeTokenSource<'a> { |
101 | fn token_kind(&self, pos: usize) -> SyntaxKind { | 314 | fn token_kind(&self, pos: usize) -> SyntaxKind { |
102 | if let Some(tok) = self.tokens.get(self.tt_pos + pos) { | 315 | if let Some(tok) = self.walker.token_idx(pos) { |
103 | tok.kind | 316 | tok.kind |
104 | } else { | 317 | } else { |
105 | SyntaxKind::EOF | 318 | SyntaxKind::EOF |
106 | } | 319 | } |
107 | } | 320 | } |
108 | fn is_token_joint_to_next(&self, pos: usize) -> bool { | 321 | fn is_token_joint_to_next(&self, pos: usize) -> bool { |
109 | self.tokens[self.tt_pos + pos].is_joint_to_next | 322 | self.walker.token_idx(pos).unwrap().is_joint_to_next |
110 | } | 323 | } |
111 | fn is_keyword(&self, pos: usize, kw: &str) -> bool { | 324 | fn is_keyword(&self, pos: usize, kw: &str) -> bool { |
112 | self.tokens[self.tt_pos + pos].text == *kw | 325 | self.walker.token_idx(pos).unwrap().text == *kw |
113 | } | 326 | } |
114 | } | 327 | } |
115 | 328 | ||
@@ -136,10 +349,6 @@ where | |||
136 | TokenPeek { iter: itertools::multipeek(iter) } | 349 | TokenPeek { iter: itertools::multipeek(iter) } |
137 | } | 350 | } |
138 | 351 | ||
139 | pub fn next(&mut self) -> Option<&tt::TokenTree> { | ||
140 | self.iter.next() | ||
141 | } | ||
142 | |||
143 | fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> { | 352 | fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> { |
144 | if p.spacing != tt::Spacing::Joint { | 353 | if p.spacing != tt::Spacing::Joint { |
145 | return None; | 354 | return None; |
@@ -162,191 +371,117 @@ where | |||
162 | } | 371 | } |
163 | } | 372 | } |
164 | 373 | ||
165 | struct TtTokenBuilder { | 374 | fn convert_multi_char_punct<'b, I>( |
166 | tokens: Vec<TtToken>, | 375 | p: &tt::Punct, |
167 | } | 376 | iter: &mut TokenPeek<'b, I>, |
168 | 377 | ) -> Option<(SyntaxKind, bool, &'static str, usize)> | |
169 | impl TtTokenBuilder { | 378 | where |
170 | fn build(sub: &tt::Subtree) -> Vec<TtToken> { | 379 | I: Iterator<Item = &'b tt::TokenTree>, |
171 | let mut res = TtTokenBuilder { tokens: vec![] }; | 380 | { |
172 | res.convert_subtree(sub); | 381 | if let Some((m, is_joint_to_next)) = iter.current_punct3(p) { |
173 | res.tokens | 382 | if let Some((kind, text)) = match m { |
174 | } | 383 | ('<', '<', '=') => Some((SHLEQ, "<<=")), |
175 | 384 | ('>', '>', '=') => Some((SHREQ, ">>=")), | |
176 | fn convert_subtree(&mut self, sub: &tt::Subtree) { | 385 | ('.', '.', '.') => Some((DOTDOTDOT, "...")), |
177 | self.push_delim(sub.delimiter, false); | 386 | ('.', '.', '=') => Some((DOTDOTEQ, "..=")), |
178 | let mut peek = TokenPeek::new(sub.token_trees.iter()); | 387 | _ => None, |
179 | while let Some(tt) = peek.iter.next() { | 388 | } { |
180 | self.convert_tt(tt, &mut peek); | 389 | return Some((kind, is_joint_to_next, text, 3)); |
181 | } | 390 | } |
182 | self.push_delim(sub.delimiter, true) | ||
183 | } | 391 | } |
184 | 392 | ||
185 | fn convert_tt<'b, I>(&mut self, tt: &tt::TokenTree, iter: &mut TokenPeek<'b, I>) | 393 | if let Some((m, is_joint_to_next)) = iter.current_punct2(p) { |
186 | where | 394 | if let Some((kind, text)) = match m { |
187 | I: Iterator<Item = &'b tt::TokenTree>, | 395 | ('<', '<') => Some((SHL, "<<")), |
188 | { | 396 | ('>', '>') => Some((SHR, ">>")), |
189 | match tt { | 397 | |
190 | tt::TokenTree::Leaf(token) => self.convert_token(token, iter), | 398 | ('|', '|') => Some((PIPEPIPE, "||")), |
191 | tt::TokenTree::Subtree(sub) => self.convert_subtree(sub), | 399 | ('&', '&') => Some((AMPAMP, "&&")), |
400 | ('%', '=') => Some((PERCENTEQ, "%=")), | ||
401 | ('*', '=') => Some((STAREQ, "*=")), | ||
402 | ('/', '=') => Some((SLASHEQ, "/=")), | ||
403 | ('^', '=') => Some((CARETEQ, "^=")), | ||
404 | |||
405 | ('&', '=') => Some((AMPEQ, "&=")), | ||
406 | ('|', '=') => Some((PIPEEQ, "|=")), | ||
407 | ('-', '=') => Some((MINUSEQ, "-=")), | ||
408 | ('+', '=') => Some((PLUSEQ, "+=")), | ||
409 | ('>', '=') => Some((GTEQ, ">=")), | ||
410 | ('<', '=') => Some((LTEQ, "<=")), | ||
411 | |||
412 | ('-', '>') => Some((THIN_ARROW, "->")), | ||
413 | ('!', '=') => Some((NEQ, "!=")), | ||
414 | ('=', '>') => Some((FAT_ARROW, "=>")), | ||
415 | ('=', '=') => Some((EQEQ, "==")), | ||
416 | ('.', '.') => Some((DOTDOT, "..")), | ||
417 | (':', ':') => Some((COLONCOLON, "::")), | ||
418 | |||
419 | _ => None, | ||
420 | } { | ||
421 | return Some((kind, is_joint_to_next, text, 2)); | ||
192 | } | 422 | } |
193 | } | 423 | } |
194 | 424 | ||
195 | fn convert_token<'b, I>(&mut self, token: &tt::Leaf, iter: &mut TokenPeek<'b, I>) | 425 | None |
196 | where | 426 | } |
197 | I: Iterator<Item = &'b tt::TokenTree>, | ||
198 | { | ||
199 | let tok = match token { | ||
200 | tt::Leaf::Literal(l) => TtToken { | ||
201 | kind: classify_literal(&l.text).unwrap().kind, | ||
202 | is_joint_to_next: false, | ||
203 | text: l.text.clone(), | ||
204 | n_tokens: 1, | ||
205 | }, | ||
206 | tt::Leaf::Punct(p) => { | ||
207 | if let Some((kind, is_joint_to_next, text, size)) = | ||
208 | Self::convert_multi_char_punct(p, iter) | ||
209 | { | ||
210 | for _ in 0..size - 1 { | ||
211 | iter.next(); | ||
212 | } | ||
213 | |||
214 | TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size } | ||
215 | } else { | ||
216 | let kind = match p.char { | ||
217 | // lexer may produce combpund tokens for these ones | ||
218 | '.' => DOT, | ||
219 | ':' => COLON, | ||
220 | '=' => EQ, | ||
221 | '!' => EXCL, | ||
222 | '-' => MINUS, | ||
223 | c => SyntaxKind::from_char(c).unwrap(), | ||
224 | }; | ||
225 | let text = { | ||
226 | let mut buf = [0u8; 4]; | ||
227 | let s: &str = p.char.encode_utf8(&mut buf); | ||
228 | SmolStr::new(s) | ||
229 | }; | ||
230 | TtToken { | ||
231 | kind, | ||
232 | is_joint_to_next: p.spacing == tt::Spacing::Joint, | ||
233 | text, | ||
234 | n_tokens: 1, | ||
235 | } | ||
236 | } | ||
237 | } | ||
238 | tt::Leaf::Ident(ident) => { | ||
239 | let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT); | ||
240 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } | ||
241 | } | ||
242 | }; | ||
243 | self.tokens.push(tok) | ||
244 | } | ||
245 | |||
246 | fn convert_multi_char_punct<'b, I>( | ||
247 | p: &tt::Punct, | ||
248 | iter: &mut TokenPeek<'b, I>, | ||
249 | ) -> Option<(SyntaxKind, bool, &'static str, usize)> | ||
250 | where | ||
251 | I: Iterator<Item = &'b tt::TokenTree>, | ||
252 | { | ||
253 | if let Some((m, is_joint_to_next)) = iter.current_punct3(p) { | ||
254 | if let Some((kind, text)) = match m { | ||
255 | ('<', '<', '=') => Some((SHLEQ, "<<=")), | ||
256 | ('>', '>', '=') => Some((SHREQ, ">>=")), | ||
257 | ('.', '.', '.') => Some((DOTDOTDOT, "...")), | ||
258 | ('.', '.', '=') => Some((DOTDOTEQ, "..=")), | ||
259 | _ => None, | ||
260 | } { | ||
261 | return Some((kind, is_joint_to_next, text, 3)); | ||
262 | } | ||
263 | } | ||
264 | 427 | ||
265 | if let Some((m, is_joint_to_next)) = iter.current_punct2(p) { | 428 | struct SubTreeWalker<'a> { |
266 | if let Some((kind, text)) = match m { | 429 | pos: usize, |
267 | ('<', '<') => Some((SHL, "<<")), | 430 | stack: Vec<(&'a tt::Subtree, Option<usize>)>, |
268 | ('>', '>') => Some((SHR, ">>")), | 431 | idx: WalkIndex, |
269 | 432 | last_steps: Vec<usize>, | |
270 | ('|', '|') => Some((PIPEPIPE, "||")), | 433 | subtree: &'a tt::Subtree, |
271 | ('&', '&') => Some((AMPAMP, "&&")), | 434 | } |
272 | ('%', '=') => Some((PERCENTEQ, "%=")), | ||
273 | ('*', '=') => Some((STAREQ, "*=")), | ||
274 | ('/', '=') => Some((SLASHEQ, "/=")), | ||
275 | ('^', '=') => Some((CARETEQ, "^=")), | ||
276 | |||
277 | ('&', '=') => Some((AMPEQ, "&=")), | ||
278 | ('|', '=') => Some((PIPEEQ, "|=")), | ||
279 | ('-', '=') => Some((MINUSEQ, "-=")), | ||
280 | ('+', '=') => Some((PLUSEQ, "+=")), | ||
281 | ('>', '=') => Some((GTEQ, ">=")), | ||
282 | ('<', '=') => Some((LTEQ, "<=")), | ||
283 | |||
284 | ('-', '>') => Some((THIN_ARROW, "->")), | ||
285 | ('!', '=') => Some((NEQ, "!=")), | ||
286 | ('=', '>') => Some((FAT_ARROW, "=>")), | ||
287 | ('=', '=') => Some((EQEQ, "==")), | ||
288 | ('.', '.') => Some((DOTDOT, "..")), | ||
289 | (':', ':') => Some((COLONCOLON, "::")), | ||
290 | |||
291 | _ => None, | ||
292 | } { | ||
293 | return Some((kind, is_joint_to_next, text, 2)); | ||
294 | } | ||
295 | } | ||
296 | 435 | ||
297 | None | 436 | fn convert_delim(d: tt::Delimiter, closing: bool) -> Option<TtToken> { |
298 | } | 437 | let (kinds, texts) = match d { |
438 | tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), | ||
439 | tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), | ||
440 | tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), | ||
441 | tt::Delimiter::None => return None, | ||
442 | }; | ||
443 | |||
444 | let idx = closing as usize; | ||
445 | let kind = kinds[idx]; | ||
446 | let text = &texts[idx..texts.len() - (1 - idx)]; | ||
447 | Some(TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 }) | ||
448 | } | ||
299 | 449 | ||
300 | fn push_delim(&mut self, d: tt::Delimiter, closing: bool) { | 450 | fn convert_literal(l: &tt::Literal) -> TtToken { |
301 | let (kinds, texts) = match d { | 451 | TtToken { |
302 | tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), | 452 | kind: classify_literal(&l.text).unwrap().kind, |
303 | tt::Delimiter::Brace => ([L_CURLY, R_CURLY], "{}"), | 453 | is_joint_to_next: false, |
304 | tt::Delimiter::Bracket => ([L_BRACK, R_BRACK], "[]"), | 454 | text: l.text.clone(), |
305 | tt::Delimiter::None => return, | 455 | n_tokens: 1, |
306 | }; | ||
307 | let idx = closing as usize; | ||
308 | let kind = kinds[idx]; | ||
309 | let text = &texts[idx..texts.len() - (1 - idx)]; | ||
310 | let tok = TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 }; | ||
311 | self.tokens.push(tok) | ||
312 | } | ||
313 | |||
314 | fn skip_sibling_leaf(leaf: &tt::Leaf, iter: &mut std::slice::Iter<tt::TokenTree>) { | ||
315 | if let tt::Leaf::Punct(p) = leaf { | ||
316 | let mut peek = TokenPeek::new(iter); | ||
317 | if let Some((_, _, _, size)) = TtTokenBuilder::convert_multi_char_punct(p, &mut peek) { | ||
318 | for _ in 0..size - 1 { | ||
319 | peek.next(); | ||
320 | } | ||
321 | } | ||
322 | } | ||
323 | } | 456 | } |
457 | } | ||
324 | 458 | ||
325 | fn count_tt_tokens( | 459 | fn convert_ident(ident: &tt::Ident) -> TtToken { |
326 | tt: &tt::TokenTree, | 460 | let kind = SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT); |
327 | iter: Option<&mut std::slice::Iter<tt::TokenTree>>, | 461 | TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } |
328 | ) -> usize { | 462 | } |
329 | match tt { | ||
330 | tt::TokenTree::Subtree(sub_tree) => { | ||
331 | let mut iter = sub_tree.token_trees.iter(); | ||
332 | let mut count = match sub_tree.delimiter { | ||
333 | tt::Delimiter::None => 0, | ||
334 | _ => 2, | ||
335 | }; | ||
336 | |||
337 | while let Some(tt) = iter.next() { | ||
338 | count += Self::count_tt_tokens(&tt, Some(&mut iter)); | ||
339 | } | ||
340 | count | ||
341 | } | ||
342 | |||
343 | tt::TokenTree::Leaf(leaf) => { | ||
344 | iter.map(|iter| { | ||
345 | Self::skip_sibling_leaf(leaf, iter); | ||
346 | }); | ||
347 | 463 | ||
348 | 1 | 464 | fn convert_punct(p: &tt::Punct, parent: &tt::Subtree, next: usize) -> TtToken { |
349 | } | 465 | let iter = parent.token_trees[next..].iter(); |
350 | } | 466 | let mut peek = TokenPeek::new(iter); |
467 | |||
468 | if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { | ||
469 | TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size } | ||
470 | } else { | ||
471 | let kind = match p.char { | ||
472 | // lexer may produce combpund tokens for these ones | ||
473 | '.' => DOT, | ||
474 | ':' => COLON, | ||
475 | '=' => EQ, | ||
476 | '!' => EXCL, | ||
477 | '-' => MINUS, | ||
478 | c => SyntaxKind::from_char(c).unwrap(), | ||
479 | }; | ||
480 | let text = { | ||
481 | let mut buf = [0u8; 4]; | ||
482 | let s: &str = p.char.encode_utf8(&mut buf); | ||
483 | SmolStr::new(s) | ||
484 | }; | ||
485 | TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 } | ||
351 | } | 486 | } |
352 | } | 487 | } |