aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--crates/ra_mbe/src/subtree_source.rs223
1 files changed, 41 insertions, 182 deletions
diff --git a/crates/ra_mbe/src/subtree_source.rs b/crates/ra_mbe/src/subtree_source.rs
index 278d046fb..8176296e6 100644
--- a/crates/ra_mbe/src/subtree_source.rs
+++ b/crates/ra_mbe/src/subtree_source.rs
@@ -45,20 +45,6 @@ impl<'a> TokenSeq<'a> {
45 } 45 }
46 } 46 }
47 } 47 }
48
49 fn len(&self) -> usize {
50 match self {
51 TokenSeq::Subtree(subtree) => subtree.token_trees.len() + 2,
52 TokenSeq::Seq(tokens) => tokens.len(),
53 }
54 }
55
56 fn child_slice(&self, pos: usize) -> &[tt::TokenTree] {
57 match self {
58 TokenSeq::Subtree(subtree) => &subtree.token_trees[pos - 1..],
59 TokenSeq::Seq(tokens) => &tokens[pos..],
60 }
61 }
62} 48}
63 49
64#[derive(Debug, Clone, Eq, PartialEq)] 50#[derive(Debug, Clone, Eq, PartialEq)]
@@ -66,7 +52,6 @@ struct TtToken {
66 pub kind: SyntaxKind, 52 pub kind: SyntaxKind,
67 pub is_joint_to_next: bool, 53 pub is_joint_to_next: bool,
68 pub text: SmolStr, 54 pub text: SmolStr,
69 pub n_tokens: usize,
70} 55}
71 56
72#[derive(Debug, Clone, Eq, PartialEq)] 57#[derive(Debug, Clone, Eq, PartialEq)]
@@ -80,19 +65,12 @@ struct SubTreeWalker<'a> {
80 pos: usize, 65 pos: usize,
81 stack: Vec<(TokenSeq<'a>, usize)>, 66 stack: Vec<(TokenSeq<'a>, usize)>,
82 cursor: WalkCursor, 67 cursor: WalkCursor,
83 last_steps: Vec<usize>,
84 ts: TokenSeq<'a>, 68 ts: TokenSeq<'a>,
85} 69}
86 70
87impl<'a> SubTreeWalker<'a> { 71impl<'a> SubTreeWalker<'a> {
88 fn new(ts: TokenSeq<'a>) -> SubTreeWalker { 72 fn new(ts: TokenSeq<'a>) -> SubTreeWalker {
89 let mut res = SubTreeWalker { 73 let mut res = SubTreeWalker { pos: 0, stack: vec![], cursor: WalkCursor::Eof, ts };
90 pos: 0,
91 stack: vec![],
92 cursor: WalkCursor::Eof,
93 last_steps: vec![],
94 ts,
95 };
96 74
97 res.reset(); 75 res.reset();
98 res 76 res
@@ -105,7 +83,6 @@ impl<'a> SubTreeWalker<'a> {
105 fn reset(&mut self) { 83 fn reset(&mut self) {
106 self.pos = 0; 84 self.pos = 0;
107 self.stack = vec![]; 85 self.stack = vec![];
108 self.last_steps = vec![];
109 86
110 self.cursor = match self.ts.get(0) { 87 self.cursor = match self.ts.get(0) {
111 DelimToken::Token(token) => match token { 88 DelimToken::Token(token) => match token {
@@ -114,10 +91,7 @@ impl<'a> SubTreeWalker<'a> {
114 self.stack.push((ts, 0)); 91 self.stack.push((ts, 0));
115 WalkCursor::Token(0, convert_delim(subtree.delimiter, false)) 92 WalkCursor::Token(0, convert_delim(subtree.delimiter, false))
116 } 93 }
117 tt::TokenTree::Leaf(leaf) => { 94 tt::TokenTree::Leaf(leaf) => WalkCursor::Token(0, convert_leaf(leaf)),
118 let next_tokens = self.ts.child_slice(0);
119 WalkCursor::Token(0, convert_leaf(&next_tokens, leaf))
120 }
121 }, 95 },
122 DelimToken::Delim(delim, is_end) => { 96 DelimToken::Delim(delim, is_end) => {
123 assert!(!is_end); 97 assert!(!is_end);
@@ -138,24 +112,6 @@ impl<'a> SubTreeWalker<'a> {
138 self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts) 112 self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts)
139 } 113 }
140 114
141 /// Move cursor backward by 1 step
142 fn backward(&mut self) {
143 if self.last_steps.is_empty() {
144 return;
145 }
146
147 self.pos -= 1;
148 let last_step = self.last_steps.pop().unwrap();
149
150 self.cursor = match self.cursor {
151 WalkCursor::Token(idx, _) => self.walk_token(idx, last_step, true),
152 WalkCursor::Eof => {
153 let len = self.top().len();
154 self.walk_token(len, last_step, true)
155 }
156 }
157 }
158
159 /// Move cursor forward by 1 step 115 /// Move cursor forward by 1 step
160 fn forward(&mut self) { 116 fn forward(&mut self) {
161 if self.is_eof() { 117 if self.is_eof() {
@@ -163,37 +119,24 @@ impl<'a> SubTreeWalker<'a> {
163 } 119 }
164 self.pos += 1; 120 self.pos += 1;
165 121
166 let step = self.current().map(|x| x.n_tokens).unwrap_or(1);
167 self.last_steps.push(step);
168
169 if let WalkCursor::Token(u, _) = self.cursor { 122 if let WalkCursor::Token(u, _) = self.cursor {
170 self.cursor = self.walk_token(u, step, false) 123 self.cursor = self.walk_token(u)
171 } 124 }
172 } 125 }
173 126
174 /// Traversal child token 127 /// Traversal child token
175 fn walk_token(&mut self, pos: usize, offset: usize, backward: bool) -> WalkCursor { 128 fn walk_token(&mut self, pos: usize) -> WalkCursor {
176 let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts); 129 let top = self.stack.last().map(|(t, _)| t).unwrap_or(&self.ts);
177 130 let pos = pos + 1;
178 if backward && pos < offset {
179 let (_, last_idx) = self.stack.pop().unwrap();
180 return self.walk_token(last_idx, offset, backward);
181 }
182
183 let pos = if backward { pos - offset } else { pos + offset };
184 131
185 match top.get(pos) { 132 match top.get(pos) {
186 DelimToken::Token(token) => match token { 133 DelimToken::Token(token) => match token {
187 tt::TokenTree::Subtree(subtree) => { 134 tt::TokenTree::Subtree(subtree) => {
188 let ts = TokenSeq::from(subtree); 135 let ts = TokenSeq::from(subtree);
189 let new_idx = if backward { ts.len() - 1 } else { 0 };
190 self.stack.push((ts, pos)); 136 self.stack.push((ts, pos));
191 WalkCursor::Token(new_idx, convert_delim(subtree.delimiter, backward)) 137 WalkCursor::Token(0, convert_delim(subtree.delimiter, false))
192 }
193 tt::TokenTree::Leaf(leaf) => {
194 let next_tokens = top.child_slice(pos);
195 WalkCursor::Token(pos, convert_leaf(&next_tokens, leaf))
196 } 138 }
139 tt::TokenTree::Leaf(leaf) => WalkCursor::Token(pos, convert_leaf(leaf)),
197 }, 140 },
198 DelimToken::Delim(delim, is_end) => { 141 DelimToken::Delim(delim, is_end) => {
199 WalkCursor::Token(pos, convert_delim(*delim, is_end)) 142 WalkCursor::Token(pos, convert_delim(*delim, is_end))
@@ -201,8 +144,7 @@ impl<'a> SubTreeWalker<'a> {
201 DelimToken::End => { 144 DelimToken::End => {
202 // it is the top level 145 // it is the top level
203 if let Some((_, last_idx)) = self.stack.pop() { 146 if let Some((_, last_idx)) = self.stack.pop() {
204 assert!(!backward); 147 self.walk_token(last_idx)
205 self.walk_token(last_idx, offset, backward)
206 } else { 148 } else {
207 WalkCursor::Eof 149 WalkCursor::Eof
208 } 150 }
@@ -237,12 +179,9 @@ impl<'a> WalkerOwner<'a> {
237 } 179 }
238 180
239 while pos >= cached.len() { 181 while pos >= cached.len() {
240 let len = cached.len(); 182 self.set_pos(cached.len());
241 cached.push({ 183 let walker = self.walker.borrow();
242 self.set_pos(len); 184 cached.push(walker.current().cloned());
243 let walker = self.walker.borrow();
244 walker.current().cloned()
245 });
246 } 185 }
247 186
248 return cached[pos].clone(); 187 return cached[pos].clone();
@@ -250,12 +189,11 @@ impl<'a> WalkerOwner<'a> {
250 189
251 fn set_pos(&self, pos: usize) { 190 fn set_pos(&self, pos: usize) {
252 let mut walker = self.walker.borrow_mut(); 191 let mut walker = self.walker.borrow_mut();
192 assert!(walker.pos <= pos);
193
253 while pos > walker.pos && !walker.is_eof() { 194 while pos > walker.pos && !walker.is_eof() {
254 walker.forward(); 195 walker.forward();
255 } 196 }
256 while pos < walker.pos {
257 walker.backward();
258 }
259 } 197 }
260 198
261 fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> { 199 fn collect_token_trees(&mut self, n: usize) -> Vec<&tt::TokenTree> {
@@ -264,15 +202,16 @@ impl<'a> WalkerOwner<'a> {
264 walker.reset(); 202 walker.reset();
265 203
266 while walker.pos < n { 204 while walker.pos < n {
267 if let WalkCursor::Token(u, tt) = &walker.cursor { 205 if let WalkCursor::Token(u, _) = &walker.cursor {
268 // We only collect the topmost child 206 // We only collect the topmost child
269 if walker.stack.len() == 0 { 207 if walker.stack.len() == 0 {
270 for i in 0..tt.n_tokens { 208 if let DelimToken::Token(token) = walker.ts.get(*u) {
271 if let DelimToken::Token(token) = walker.ts.get(u + i) { 209 res.push(token);
272 res.push(token);
273 }
274 } 210 }
275 } else if walker.stack.len() == 1 { 211 }
212 // Check whether the second level is a subtree
213 // if so, collect its parent which is topmost child
214 else if walker.stack.len() == 1 {
276 if let DelimToken::Delim(_, is_end) = walker.top().get(*u) { 215 if let DelimToken::Delim(_, is_end) = walker.top().get(*u) {
277 if !is_end { 216 if !is_end {
278 let (_, last_idx) = &walker.stack[0]; 217 let (_, last_idx) = &walker.stack[0];
@@ -343,78 +282,6 @@ impl<'a> TokenSource for SubtreeTokenSource<'a> {
343 } 282 }
344} 283}
345 284
346pub(crate) struct TokenPeek<'a, I>
347where
348 I: Iterator<Item = &'a tt::TokenTree>,
349{
350 iter: itertools::MultiPeek<I>,
351}
352
353// helper function
354fn to_punct(tt: &tt::TokenTree) -> Option<&tt::Punct> {
355 if let tt::TokenTree::Leaf(tt::Leaf::Punct(pp)) = tt {
356 return Some(pp);
357 }
358 None
359}
360
361impl<'a, I> TokenPeek<'a, I>
362where
363 I: Iterator<Item = &'a tt::TokenTree>,
364{
365 pub fn new(iter: I) -> Self {
366 TokenPeek { iter: itertools::multipeek(iter) }
367 }
368
369 pub fn current_punct2(&mut self, p: &tt::Punct) -> Option<((char, char), bool)> {
370 if p.spacing != tt::Spacing::Joint {
371 return None;
372 }
373
374 self.iter.reset_peek();
375 let p1 = to_punct(self.iter.peek()?)?;
376 Some(((p.char, p1.char), p1.spacing == tt::Spacing::Joint))
377 }
378
379 pub fn current_punct3(&mut self, p: &tt::Punct) -> Option<((char, char, char), bool)> {
380 self.current_punct2(p).and_then(|((p0, p1), last_joint)| {
381 if !last_joint {
382 None
383 } else {
384 let p2 = to_punct(*self.iter.peek()?)?;
385 Some(((p0, p1, p2.char), p2.spacing == tt::Spacing::Joint))
386 }
387 })
388 }
389}
390
391// FIXME: Remove this function
392fn convert_multi_char_punct<'b, I>(
393 p: &tt::Punct,
394 iter: &mut TokenPeek<'b, I>,
395) -> Option<(SyntaxKind, bool, &'static str, usize)>
396where
397 I: Iterator<Item = &'b tt::TokenTree>,
398{
399 if let Some((m, is_joint_to_next)) = iter.current_punct3(p) {
400 if let Some((kind, text)) = match m {
401 _ => None,
402 } {
403 return Some((kind, is_joint_to_next, text, 3));
404 }
405 }
406
407 if let Some((m, is_joint_to_next)) = iter.current_punct2(p) {
408 if let Some((kind, text)) = match m {
409 _ => None,
410 } {
411 return Some((kind, is_joint_to_next, text, 2));
412 }
413 }
414
415 None
416}
417
418fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken { 285fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken {
419 let (kinds, texts) = match d { 286 let (kinds, texts) = match d {
420 tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"), 287 tt::Delimiter::Parenthesis => ([L_PAREN, R_PAREN], "()"),
@@ -426,7 +293,7 @@ fn convert_delim(d: tt::Delimiter, closing: bool) -> TtToken {
426 let idx = closing as usize; 293 let idx = closing as usize;
427 let kind = kinds[idx]; 294 let kind = kinds[idx];
428 let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" }; 295 let text = if texts.len() > 0 { &texts[idx..texts.len() - (1 - idx)] } else { "" };
429 TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text), n_tokens: 1 } 296 TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
430} 297}
431 298
432fn convert_literal(l: &tt::Literal) -> TtToken { 299fn convert_literal(l: &tt::Literal) -> TtToken {
@@ -437,7 +304,7 @@ fn convert_literal(l: &tt::Literal) -> TtToken {
437 _ => panic!("Fail to convert given literal {:#?}", &l), 304 _ => panic!("Fail to convert given literal {:#?}", &l),
438 }); 305 });
439 306
440 TtToken { kind, is_joint_to_next: false, text: l.text.clone(), n_tokens: 1 } 307 TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
441} 308}
442 309
443fn convert_ident(ident: &tt::Ident) -> TtToken { 310fn convert_ident(ident: &tt::Ident) -> TtToken {
@@ -447,39 +314,31 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
447 SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT) 314 SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT)
448 }; 315 };
449 316
450 TtToken { kind, is_joint_to_next: false, text: ident.text.clone(), n_tokens: 1 } 317 TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
451} 318}
452 319
453fn convert_punct(p: &tt::Punct, next_tokens: &[tt::TokenTree]) -> TtToken { 320fn convert_punct(p: &tt::Punct) -> TtToken {
454 let mut iter = next_tokens.iter(); 321 let kind = match p.char {
455 iter.next(); 322 // lexer may produce combpund tokens for these ones
456 let mut peek = TokenPeek::new(iter); 323 '.' => DOT,
457 324 ':' => COLON,
458 if let Some((kind, is_joint_to_next, text, size)) = convert_multi_char_punct(p, &mut peek) { 325 '=' => EQ,
459 TtToken { kind, is_joint_to_next, text: text.into(), n_tokens: size } 326 '!' => EXCL,
460 } else { 327 '-' => MINUS,
461 let kind = match p.char { 328 c => SyntaxKind::from_char(c).unwrap(),
462 // lexer may produce combpund tokens for these ones 329 };
463 '.' => DOT, 330 let text = {
464 ':' => COLON, 331 let mut buf = [0u8; 4];
465 '=' => EQ, 332 let s: &str = p.char.encode_utf8(&mut buf);
466 '!' => EXCL, 333 SmolStr::new(s)
467 '-' => MINUS, 334 };
468 c => SyntaxKind::from_char(c).unwrap(), 335 TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
469 };
470 let text = {
471 let mut buf = [0u8; 4];
472 let s: &str = p.char.encode_utf8(&mut buf);
473 SmolStr::new(s)
474 };
475 TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text, n_tokens: 1 }
476 }
477} 336}
478 337
479fn convert_leaf(tokens: &[tt::TokenTree], leaf: &tt::Leaf) -> TtToken { 338fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
480 match leaf { 339 match leaf {
481 tt::Leaf::Literal(l) => convert_literal(l), 340 tt::Leaf::Literal(l) => convert_literal(l),
482 tt::Leaf::Ident(ident) => convert_ident(ident), 341 tt::Leaf::Ident(ident) => convert_ident(ident),
483 tt::Leaf::Punct(punct) => convert_punct(punct, tokens), 342 tt::Leaf::Punct(punct) => convert_punct(punct),
484 } 343 }
485} 344}