aboutsummaryrefslogtreecommitdiff
path: root/crates/ra_mbe/src
diff options
context:
space:
mode:
authorEdwin Cheng <[email protected]>2020-03-20 19:04:11 +0000
committerEdwin Cheng <[email protected]>2020-03-20 19:08:56 +0000
commit622c843a4c598befaf459e64e9f75f31b4886a5b (patch)
tree6a46953e52aeb02f724fd87afe9fab2cec1c7691 /crates/ra_mbe/src
parent27c516970bea6bee9d6bca52ac9ad619412809ef (diff)
Add TokenConvertor trait
Diffstat (limited to 'crates/ra_mbe/src')
-rw-r--r--crates/ra_mbe/src/syntax_bridge.rs388
-rw-r--r--crates/ra_mbe/src/tests.rs6
2 files changed, 155 insertions, 239 deletions
diff --git a/crates/ra_mbe/src/syntax_bridge.rs b/crates/ra_mbe/src/syntax_bridge.rs
index 0678c37ee..540afc87c 100644
--- a/crates/ra_mbe/src/syntax_bridge.rs
+++ b/crates/ra_mbe/src/syntax_bridge.rs
@@ -3,12 +3,11 @@
3use ra_parser::{FragmentKind, ParseError, TreeSink}; 3use ra_parser::{FragmentKind, ParseError, TreeSink};
4use ra_syntax::{ 4use ra_syntax::{
5 ast::{self, make::tokens::doc_comment}, 5 ast::{self, make::tokens::doc_comment},
6 tokenize, AstToken, NodeOrToken, Parse, SmolStr, SyntaxKind, 6 tokenize, AstToken, Parse, SmolStr, SyntaxKind,
7 SyntaxKind::*, 7 SyntaxKind::*,
8 SyntaxNode, SyntaxTreeBuilder, TextRange, TextUnit, Token, T, 8 SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextUnit, Token as RawToken, T,
9}; 9};
10use rustc_hash::FxHashMap; 10use rustc_hash::FxHashMap;
11use std::iter::successors;
12use tt::buffer::{Cursor, TokenBuffer}; 11use tt::buffer::{Cursor, TokenBuffer};
13 12
14use crate::subtree_source::SubtreeTokenSource; 13use crate::subtree_source::SubtreeTokenSource;
@@ -50,10 +49,8 @@ pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenM
50/// will consume). 49/// will consume).
51pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> { 50pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> {
52 let global_offset = node.text_range().start(); 51 let global_offset = node.text_range().start();
53 let mut c = Convertor { 52 let mut c = Convertor::new(node, global_offset);
54 id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } }, 53 let subtree = c.go()?;
55 };
56 let subtree = c.go(node)?;
57 Some((subtree, c.id_alloc.map)) 54 Some((subtree, c.id_alloc.map))
58} 55}
59 56
@@ -237,16 +234,6 @@ impl TokenIdAlloc {
237 token_id 234 token_id
238 } 235 }
239 236
240 fn delim(&mut self, open_abs_range: TextRange, close_abs_range: TextRange) -> tt::TokenId {
241 let open_relative_range = open_abs_range - self.global_offset;
242 let close_relative_range = close_abs_range - self.global_offset;
243 let token_id = tt::TokenId(self.next_id);
244 self.next_id += 1;
245
246 self.map.insert_delim(token_id, open_relative_range, close_relative_range);
247 token_id
248 }
249
250 fn open_delim(&mut self, open_abs_range: TextRange) -> tt::TokenId { 237 fn open_delim(&mut self, open_abs_range: TextRange) -> tt::TokenId {
251 let token_id = tt::TokenId(self.next_id); 238 let token_id = tt::TokenId(self.next_id);
252 self.next_id += 1; 239 self.next_id += 1;
@@ -264,15 +251,19 @@ struct RawConvertor<'a> {
264 text: &'a str, 251 text: &'a str,
265 offset: TextUnit, 252 offset: TextUnit,
266 id_alloc: TokenIdAlloc, 253 id_alloc: TokenIdAlloc,
267 inner: std::slice::Iter<'a, Token>, 254 inner: std::slice::Iter<'a, RawToken>,
268} 255}
269 256
270trait SrcToken { 257trait SrcToken {
271 fn kind() -> SyntaxKind; 258 fn kind(&self) -> SyntaxKind;
259
260 fn to_char(&self) -> Option<char>;
261
262 fn to_text(&self) -> SmolStr;
272} 263}
273 264
274trait TokenConvertor { 265trait TokenConvertor {
275 type Token : SrcToken; 266 type Token: SrcToken;
276 267
277 fn go(&mut self) -> Option<tt::Subtree> { 268 fn go(&mut self) -> Option<tt::Subtree> {
278 let mut subtree = tt::Subtree::default(); 269 let mut subtree = tt::Subtree::default();
@@ -291,10 +282,6 @@ trait TokenConvertor {
291 Some(subtree) 282 Some(subtree)
292 } 283 }
293 284
294 fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
295
296 fn peek(&self) -> Option<Self::Token>;
297
298 fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) { 285 fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
299 let (token, range) = match self.bump() { 286 let (token, range) = match self.bump() {
300 None => return, 287 None => return,
@@ -303,8 +290,7 @@ trait TokenConvertor {
303 290
304 let k: SyntaxKind = token.kind(); 291 let k: SyntaxKind = token.kind();
305 if k == COMMENT { 292 if k == COMMENT {
306 let node = doc_comment(&self.text[range]); 293 if let Some(tokens) = self.convert_doc_comment(&token) {
307 if let Some(tokens) = convert_doc_comment(&node) {
308 result.extend(tokens); 294 result.extend(tokens);
309 } 295 }
310 return; 296 return;
@@ -320,40 +306,39 @@ trait TokenConvertor {
320 306
321 if let Some((kind, closed)) = delim { 307 if let Some((kind, closed)) = delim {
322 let mut subtree = tt::Subtree::default(); 308 let mut subtree = tt::Subtree::default();
323 let id = self.id_alloc.open_delim(range); 309 let id = self.id_alloc().open_delim(range);
324 subtree.delimiter = Some(tt::Delimiter { kind, id }); 310 subtree.delimiter = Some(tt::Delimiter { kind, id });
325 311
326 while self.peek().map(|it| it.kind != closed).unwrap_or(false) { 312 while self.peek().map(|it| it.kind() != closed).unwrap_or(false) {
327 self.collect_leaf(&mut subtree.token_trees); 313 self.collect_leaf(&mut subtree.token_trees);
328 } 314 }
329 let last_range = match self.bump() { 315 let last_range = match self.bump() {
330 None => return, 316 None => return,
331 Some(it) => it.1, 317 Some(it) => it.1,
332 }; 318 };
333 self.id_alloc.close_delim(id, last_range); 319 self.id_alloc().close_delim(id, last_range);
334 subtree.into() 320 subtree.into()
335 } else { 321 } else {
336 let spacing = match self.peek() { 322 let spacing = match self.peek() {
337 Some(next) 323 Some(next)
338 if next.kind.is_trivia() 324 if next.kind().is_trivia()
339 || next.kind == T!['['] 325 || next.kind() == T!['[']
340 || next.kind == T!['{'] 326 || next.kind() == T!['{']
341 || next.kind == T!['('] => 327 || next.kind() == T!['('] =>
342 { 328 {
343 tt::Spacing::Alone 329 tt::Spacing::Alone
344 } 330 }
345 Some(next) if next.kind.is_punct() => tt::Spacing::Joint, 331 Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
346 _ => tt::Spacing::Alone, 332 _ => tt::Spacing::Alone,
347 }; 333 };
348 let char = 334 let char = token.to_char().expect("Token from lexer must be single char");
349 self.text[range].chars().next().expect("Token from lexer must be single char");
350 335
351 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into() 336 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
352 } 337 }
353 } else { 338 } else {
354 macro_rules! make_leaf { 339 macro_rules! make_leaf {
355 ($i:ident) => { 340 ($i:ident) => {
356 tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into() 341 tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into()
357 }; 342 };
358 } 343 }
359 let leaf: tt::Leaf = match k { 344 let leaf: tt::Leaf = match k {
@@ -367,237 +352,168 @@ trait TokenConvertor {
367 leaf.into() 352 leaf.into()
368 }); 353 });
369 } 354 }
355
356 fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>;
357
358 fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
359
360 fn peek(&self) -> Option<Self::Token>;
361
362 fn id_alloc(&mut self) -> &mut TokenIdAlloc;
370} 363}
371 364
372impl RawConvertor<'_> { 365impl<'a> SrcToken for (RawToken, &'a str) {
373 fn go(&mut self) -> Option<tt::Subtree> { 366 fn kind(&self) -> SyntaxKind {
374 let mut subtree = tt::Subtree::default(); 367 self.0.kind
375 subtree.delimiter = None;
376 while self.peek().is_some() {
377 self.collect_leaf(&mut subtree.token_trees);
378 }
379 if subtree.token_trees.is_empty() {
380 return None;
381 }
382 if subtree.token_trees.len() == 1 {
383 if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
384 return Some(first.clone());
385 }
386 }
387 Some(subtree)
388 } 368 }
389 369
390 fn bump(&mut self) -> Option<(Token, TextRange)> { 370 fn to_char(&self) -> Option<char> {
391 let token = self.inner.next()?; 371 self.1.chars().next()
392 let range = TextRange::offset_len(self.offset, token.len);
393 self.offset += token.len;
394 Some((*token, range))
395 } 372 }
396 373
397 fn peek(&self) -> Option<Token> { 374 fn to_text(&self) -> SmolStr {
398 self.inner.as_slice().get(0).cloned() 375 self.1.into()
399 } 376 }
400 377}
401 378
402 fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) { 379impl RawConvertor<'_> {}
403 let (token, range) = match self.bump() {
404 None => return,
405 Some(it) => it,
406 };
407 380
408 let k: SyntaxKind = token.kind; 381impl<'a> TokenConvertor for RawConvertor<'a> {
409 if k == COMMENT { 382 type Token = (RawToken, &'a str);
410 let node = doc_comment(&self.text[range]);
411 if let Some(tokens) = convert_doc_comment(&node) {
412 result.extend(tokens);
413 }
414 return;
415 }
416 383
417 result.push(if k.is_punct() { 384 fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
418 let delim = match k { 385 convert_doc_comment(&doc_comment(token.1))
419 T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])), 386 }
420 T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
421 T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
422 _ => None,
423 };
424 387
425 if let Some((kind, closed)) = delim { 388 fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
426 let mut subtree = tt::Subtree::default(); 389 let token = self.inner.next()?;
427 let id = self.id_alloc.open_delim(range); 390 let range = TextRange::offset_len(self.offset, token.len);
428 subtree.delimiter = Some(tt::Delimiter { kind, id }); 391 self.offset += token.len;
429 392
430 while self.peek().map(|it| it.kind != closed).unwrap_or(false) { 393 Some(((*token, &self.text[range]), range))
431 self.collect_leaf(&mut subtree.token_trees); 394 }
432 }
433 let last_range = match self.bump() {
434 None => return,
435 Some(it) => it.1,
436 };
437 self.id_alloc.close_delim(id, last_range);
438 subtree.into()
439 } else {
440 let spacing = match self.peek() {
441 Some(next)
442 if next.kind.is_trivia()
443 || next.kind == T!['[']
444 || next.kind == T!['{']
445 || next.kind == T!['('] =>
446 {
447 tt::Spacing::Alone
448 }
449 Some(next) if next.kind.is_punct() => tt::Spacing::Joint,
450 _ => tt::Spacing::Alone,
451 };
452 let char =
453 self.text[range].chars().next().expect("Token from lexer must be single char");
454 395
455 tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc.alloc(range) }).into() 396 fn peek(&self) -> Option<Self::Token> {
456 } 397 let token = self.inner.as_slice().get(0).cloned();
457 } else {
458 macro_rules! make_leaf {
459 ($i:ident) => {
460 tt::$i { id: self.id_alloc.alloc(range), text: self.text[range].into() }.into()
461 };
462 }
463 let leaf: tt::Leaf = match k {
464 T![true] | T![false] => make_leaf!(Literal),
465 IDENT | LIFETIME => make_leaf!(Ident),
466 k if k.is_keyword() => make_leaf!(Ident),
467 k if k.is_literal() => make_leaf!(Literal),
468 _ => return,
469 };
470 398
471 leaf.into() 399 token.map(|it| {
472 }); 400 let range = TextRange::offset_len(self.offset, it.len);
401 (it, &self.text[range])
402 })
403 }
404
405 fn id_alloc(&mut self) -> &mut TokenIdAlloc {
406 &mut self.id_alloc
473 } 407 }
474} 408}
475 409
476// FIXME: There are some duplicate logic between RawConvertor and Convertor
477// It would be nice to refactor to converting SyntaxNode to ra_parser::Token and thus
478// use RawConvertor directly. But performance-wise it may not be a good idea ?
479struct Convertor { 410struct Convertor {
480 id_alloc: TokenIdAlloc, 411 id_alloc: TokenIdAlloc,
412 current: Option<SyntaxToken>,
413 range: TextRange,
414 punct_offset: Option<(SyntaxToken, TextUnit)>,
481} 415}
482 416
483impl Convertor { 417impl Convertor {
484 fn go(&mut self, tt: &SyntaxNode) -> Option<tt::Subtree> { 418 fn new(node: &SyntaxNode, global_offset: TextUnit) -> Convertor {
485 // This tree is empty 419 Convertor {
486 if tt.first_child_or_token().is_none() { 420 id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
487 return Some(tt::Subtree { token_trees: vec![], delimiter: None }); 421 current: node.first_token(),
422 range: node.text_range(),
423 punct_offset: None,
488 } 424 }
425 }
426}
489 427
490 let first_child = tt.first_child_or_token()?; 428enum SynToken {
491 let last_child = tt.last_child_or_token()?; 429 Ordiniary(SyntaxToken),
430 Punch(SyntaxToken, TextUnit),
431}
492 432
493 // ignore trivial first_child and last_child 433impl SynToken {
494 let first_child = successors(Some(first_child), |it| { 434 fn token(&self) -> &SyntaxToken {
495 if it.kind().is_trivia() { 435 match self {
496 it.next_sibling_or_token() 436 SynToken::Ordiniary(it) => it,
497 } else { 437 SynToken::Punch(it, _) => it,
498 None
499 }
500 })
501 .last()
502 .unwrap();
503 if first_child.kind().is_trivia() {
504 return Some(tt::Subtree { token_trees: vec![], delimiter: None });
505 } 438 }
439 }
440}
506 441
507 let last_child = successors(Some(last_child), |it| { 442impl SrcToken for SynToken {
508 if it.kind().is_trivia() { 443 fn kind(&self) -> SyntaxKind {
509 it.prev_sibling_or_token() 444 self.token().kind()
510 } else { 445 }
511 None 446 fn to_char(&self) -> Option<char> {
512 } 447 match self {
513 }) 448 SynToken::Ordiniary(_) => None,
514 .last() 449 SynToken::Punch(it, i) => it.text().chars().nth(i.to_usize()),
515 .unwrap(); 450 }
516 451 }
517 let (delimiter_kind, skip_first) = match (first_child.kind(), last_child.kind()) { 452 fn to_text(&self) -> SmolStr {
518 (T!['('], T![')']) => (Some(tt::DelimiterKind::Parenthesis), true), 453 self.token().text().clone()
519 (T!['{'], T!['}']) => (Some(tt::DelimiterKind::Brace), true), 454 }
520 (T!['['], T![']']) => (Some(tt::DelimiterKind::Bracket), true), 455}
521 _ => (None, false), 456
522 }; 457impl TokenConvertor for Convertor {
523 let delimiter = delimiter_kind.map(|kind| tt::Delimiter { 458 type Token = SynToken;
524 kind, 459 fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
525 id: self.id_alloc.delim(first_child.text_range(), last_child.text_range()), 460 convert_doc_comment(token.token())
526 }); 461 }
527 462
528 let mut token_trees = Vec::new(); 463 fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
529 let mut child_iter = tt.children_with_tokens().skip(skip_first as usize).peekable(); 464 let curr = self.current.clone()?;
465 if !curr.text_range().is_subrange(&self.range) {
466 return None;
467 }
530 468
531 while let Some(child) = child_iter.next() { 469 if let Some((punct, offset)) = self.punct_offset.clone() {
532 if skip_first && (child == first_child || child == last_child) { 470 if offset.to_usize() + 1 < punct.text().len() {
533 continue; 471 let offset = offset + TextUnit::from_usize(1);
472 let range = punct.text_range();
473 self.punct_offset = Some((punct, offset));
474 let range = TextRange::offset_len(range.start() + offset, TextUnit::from_usize(1));
475 return Some((SynToken::Punch(curr, offset), range));
534 } 476 }
477 }
535 478
536 match child { 479 self.current = curr.next_token();
537 NodeOrToken::Token(token) => {
538 if let Some(doc_tokens) = convert_doc_comment(&token) {
539 token_trees.extend(doc_tokens);
540 } else if token.kind().is_trivia() {
541 continue;
542 } else if token.kind().is_punct() {
543 // we need to pull apart joined punctuation tokens
544 let last_spacing = match child_iter.peek() {
545 Some(NodeOrToken::Token(token)) => {
546 if token.kind().is_punct() {
547 tt::Spacing::Joint
548 } else {
549 tt::Spacing::Alone
550 }
551 }
552 _ => tt::Spacing::Alone,
553 };
554 let spacing_iter = std::iter::repeat(tt::Spacing::Joint)
555 .take(token.text().len() - 1)
556 .chain(std::iter::once(last_spacing));
557 for (char, spacing) in token.text().chars().zip(spacing_iter) {
558 token_trees.push(
559 tt::Leaf::from(tt::Punct {
560 char,
561 spacing,
562 id: self.id_alloc.alloc(token.text_range()),
563 })
564 .into(),
565 );
566 }
567 } else {
568 macro_rules! make_leaf {
569 ($i:ident) => {
570 tt::$i {
571 id: self.id_alloc.alloc(token.text_range()),
572 text: token.text().clone(),
573 }
574 .into()
575 };
576 }
577 480
578 let child: tt::Leaf = match token.kind() { 481 let token = if curr.kind().is_punct() {
579 T![true] | T![false] => make_leaf!(Literal), 482 let range = curr.text_range();
580 IDENT | LIFETIME => make_leaf!(Ident), 483 self.punct_offset = Some((curr.clone(), TextUnit::from_usize(0)));
581 k if k.is_keyword() => make_leaf!(Ident), 484 (SynToken::Punch(curr, TextUnit::from_usize(0)), range)
582 k if k.is_literal() => make_leaf!(Literal), 485 } else {
583 _ => return None, 486 self.punct_offset = None;
584 }; 487 let range = curr.text_range();
585 token_trees.push(child.into()); 488 (SynToken::Ordiniary(curr), range)
586 } 489 };
587 } 490
588 NodeOrToken::Node(node) => { 491 Some(token)
589 let child_subtree = self.go(&node)?; 492 }
590 if child_subtree.delimiter.is_none() && node.kind() != SyntaxKind::TOKEN_TREE { 493
591 token_trees.extend(child_subtree.token_trees); 494 fn peek(&self) -> Option<Self::Token> {
592 } else { 495 let curr = self.current.clone()?;
593 token_trees.push(child_subtree.into()); 496 if !curr.text_range().is_subrange(&self.range) {
594 } 497 return None;
595 } 498 }
596 }; 499
500 if let Some((punct, mut offset)) = self.punct_offset.clone() {
501 offset = offset + TextUnit::from_usize(1);
502 if offset.to_usize() < punct.text().len() {
503 return Some(SynToken::Punch(punct, offset));
504 }
597 } 505 }
598 506
599 let res = tt::Subtree { delimiter, token_trees }; 507 let token = if curr.kind().is_punct() {
600 Some(res) 508 SynToken::Punch(curr, TextUnit::from_usize(0))
509 } else {
510 SynToken::Ordiniary(curr)
511 };
512 Some(token)
513 }
514
515 fn id_alloc(&mut self) -> &mut TokenIdAlloc {
516 &mut self.id_alloc
601 } 517 }
602} 518}
603 519
diff --git a/crates/ra_mbe/src/tests.rs b/crates/ra_mbe/src/tests.rs
index 966af1d12..a3f242e49 100644
--- a/crates/ra_mbe/src/tests.rs
+++ b/crates/ra_mbe/src/tests.rs
@@ -1449,8 +1449,8 @@ impl MacroFixture {
1449 let macro_invocation = 1449 let macro_invocation =
1450 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap(); 1450 source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
1451 1451
1452 let (invocation_tt, _) = 1452 let (invocation_tt, _) = ast_to_token_tree(&macro_invocation.token_tree().unwrap())
1453 ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap(); 1453 .ok_or_else(|| ExpandError::ConversionError)?;
1454 1454
1455 self.rules.expand(&invocation_tt).result() 1455 self.rules.expand(&invocation_tt).result()
1456 } 1456 }
@@ -1694,5 +1694,5 @@ fn test_expand_bad_literal() {
1694 macro_rules! foo { ($i:literal) => {}; } 1694 macro_rules! foo { ($i:literal) => {}; }
1695 "#, 1695 "#,
1696 ) 1696 )
1697 .assert_expand_err(r#"foo!(&k");"#, &ExpandError::BindingError("".to_string())); 1697 .assert_expand_err(r#"foo!(&k");"#, &ExpandError::ConversionError);
1698} 1698}